<commit_before># -*- coding: utf-8 -*-
"""OpenBMP MRT

  Copyright (c) 2013-2016 Cisco Systems, Inc. and others.  All rights reserved.
  This program and the accompanying materials are made available under the
  terms of the Eclipse Public License v1.0 which accompanies this distribution,
  and is available at http://www.eclipse.org/legal/epl-v10.html

  .. moduleauthor:: Tim Evens <tievens@cisco.com>
"""
import multiprocessing
import socket
import kafka
import kafka.common
import time

from openbmp.mrt_bgp4mp_object import MrtBgpMpObject
from openbmp.logger import init_mp_logger

from openbmp.api.parsed.message import Message
from openbmp.api.parsed.message import MsgBusFields
from openbmp.api.parsed.message import Collector
from openbmp.api.parsed.message import Router
from openbmp.api.parsed.message import Peer

from openbmp.bmp import bmp_parse_bmphdr, bmp_parse_peerhdr

class BMPConsumer(multiprocessing.Process):
    """ OpenBMP consumer

        Consumes openbmp.parsed.collector, openbmp.parsed.router, and openbmp.bmp_raw
        message feeds from Kafka.  Collector and router messages are cached.
    """
    # Memory cache of collectors
    COLLECTORS = {}

    # Memory cache of routers
    ROUTERS = {}

    # Memory cache of peers
    PEERS = {}

    def __init__(self, cfg, mrt_bgp4mp_queue, log_queue):
        """ Constructor

            :param cfg:               Configuration dictionary
            :param mrt_bgp4mp_queue:  Output for MRT BGP4MP type
            :param log_queue:         Logging queue - sync logging
        """
        multiprocessing.Process.__init__(self)
        self._stop = multiprocessing.Event()

        self._cfg = cfg
        self._bgp4mp_queue = mrt_bgp4mp_queue
        self._log_queue = log_queue
        self.LOG = None

    def run(self):
        """ Override """
        self.LOG = init_mp_logger("bmp_consumer", self._log_queue)

        # Enable to topics/feeds
        topics = ['openbmp.parsed.collector', 'openbmp.parsed.router', 'openbmp.parsed.peer', 'openbmp.bmp_raw']

        self.LOG.info("Running bmp_consumer")

        # wait for config to load
        while not self.stopped():
            if self._cfg and 'kafka' in self._cfg:
                break

        try:
            # connect and bind to topics
            self.LOG.info("Connecting to %s ... takes a minute to load offsets and topics, please wait" % self._cfg['kafka']['servers'])
            consumer = kafka.KafkaConsumer(*topics,
                                           bootstrap_servers=self._cfg['kafka']['servers'],
                                           client_id=self._cfg['kafka']['client_id'] + '-' + socket.gethostname(),
                                           socket_timeout_ms=5000,
                                           fetch_message_max_bytes=5000000,
                                           group_id=self._cfg['kafka']['group_id'],
                                           auto_commit_enable=True,
                                           auto_commit_interval_ms=1000,
                                           auto_offset_reset='largest' if
                                                            self._cfg['kafka']['offset_reset_largest'] else "smallest")

            self.LOG.info("Connected, now consuming")

            prev_ts = time.time()

            # Loop till stopped
            while not self.stopped():

                # Read messages
                for m in consumer.fetch_messages():
                    if self.stopped():
                        break

                    self.process_msg(m)
                    consumer.task_done(m)

        except kafka.common.KafkaUnavailableError as err:
            self.LOG.error("Kafka Error: %s" % str(err))

        except KeyboardInterrupt:
            pass

        self.LOG.info("consumer stopped")

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.is_set()

    def process_msg(self, msg):
        """ Process the message

            :param msg:     Message consumed
        """

        m = Message(msg.value)  # Gets body of kafka message.

        if msg.topic == 'openbmp.parsed.collector':
            self.process_collector_msg(m)

        elif msg.topic == 'openbmp.parsed.router':
            self.process_router_msg(m)

        elif msg.topic == 'openbmp.parsed.peer':
            self.process_peer_msg(m)

        elif msg.topic == 'openbmp.bmp_raw':
            self.process_bmp_raw_msg(m)

    def process_collector_msg(self, msg):
        """ Process collector message and cache info for use later

            :param msg:      Message object
        """
        obj = Collector(msg)

        map_list = obj.getRowMap()

        for row in map_list:
            if len(row):
                try:
                    self.LOG.debug("collector: %s [%s]", row[MsgBusFields.ADMIN_ID.getName()], row[MsgBusFields.ACTION.getName()])

                    if row[MsgBusFields.ACTION.getName()] in ('started', 'heartbeat', 'change'):

                        # Update collector hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.COLLECTORS:
                            self.COLLECTORS[row[MsgBusFields.HASH.getName()]] = {'admin_id': row[MsgBusFields.ADMIN_ID.getName()]}

                    else:
                        self.COLLECTORS.pop(row[MsgBusFields.HASH.getName()], None)

                except:
                    pass

    def process_router_msg(self, msg):
        """ Process Router message

            :param msg:      Message object
        """
        obj = Router(msg)

        map_list = obj.getRowMap()

        # Log messages
        for row in map_list:
            if len(row):
                try:
                    self.LOG.info("router: [%s] %s %s", row[MsgBusFields.ACTION.getName()],
                                  row[MsgBusFields.NAME.getName()], row[MsgBusFields.IP_ADDRESS.getName()])

                    if row[MsgBusFields.ACTION.getName()] in ('first', 'init'):
                        # Update the router hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.ROUTERS:
                            self.ROUTERS[row[MsgBusFields.HASH.getName()]] = {'ip': msg.getRouterIp(), 'name': row[MsgBusFields.NAME.getName()]}
                    # else:
                    #     self.ROUTERS.pop(row[MsgBusFields.HASH.getName()], None)

                except:
                    self.LOG.debug("router parse error")
                    pass

    def process_peer_msg(self, msg):
        """ Process Peer message

            :param msg:      Message object
        """

        obj = Peer(msg)

        map_list = obj.getRowMap()

        # Log messages
        for row in map_list:
            if len(row):
                try:
                    peer_key = row[MsgBusFields.ROUTER_HASH.getName()] + '_' + row[MsgBusFields.REMOTE_BGP_ID.getName()] + '_' + row[MsgBusFields.PEER_RD.getName()]

                    self.LOG.info("peer: [%s] %s %s", row[MsgBusFields.ACTION.getName()],
                                  row[MsgBusFields.NAME.getName()], row[MsgBusFields.REMOTE_IP.getName()]) # Peer IP address

                    if row[MsgBusFields.ACTION.getName()] in ('up'):

                        asn_len = 2
                        if ('4 Octet ASN' in row[MsgBusFields.ADV_CAP.getName()] and
                            '4 Octet ASN' in row[MsgBusFields.RECV_CAP.getName()]):
                            asn_len = 4

                        # Update the peer hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.PEERS:
                            self.PEERS[peer_key] = {'remote_ip': row[MsgBusFields.REMOTE_IP.getName()],
                                                    'remote_asn': row[MsgBusFields.REMOTE_ASN.getName()],
                                                    'local_ip': row[MsgBusFields.LOCAL_IP.getName()],
                                                    'local_asn': row[MsgBusFields.LOCAL_ASN.getName()],
                                                    'asn_len': asn_len,
                                                    'rd': row[MsgBusFields.PEER_RD.getName()]}
                except:
                    self.LOG.debug("peer parse error")
                    pass

    def process_bmp_raw_msg(self, msg):
        """ Process BMP RAW message

        :param msg:      Message object
        """

        c_hash = msg.getCollector_hash_id() # Collector Hash ID
        r_hash = msg.getRouter_hash_id() # Router Hash ID
        r_ip = msg.getRouterIp() # Router IP address
        data = msg.getContent() # Message data to be consumed (should not contain headers)

        mrt_bgp_msg = MrtBgpMpObject()

        mrt_bgp_msg.COLLECTOR_HASH_ID = c_hash
        mrt_bgp_msg.ROUTER_IP = r_ip

        mrt_bgp_msg.PEER_KEY = ""

        # Parse the BMP header
        bmp_hdrs = bmp_parse_bmphdr(data)

        # Parse the BMP headers
        if bmp_hdrs['type'] in ('ROUTE_MON'):
            peer_hdr = bmp_parse_peerhdr(data[6:])
            mrt_bgp_msg.BMP_DATA = data[48:]

            mrt_bgp_msg.PEER_ADDR = peer_hdr['addr']
            mrt_bgp_msg.PEER_RD = peer_hdr['dist_id']
            mrt_bgp_msg.PEER_ASN = peer_hdr['asn']
            mrt_bgp_msg.PEER_BGP_ID = peer_hdr['bgp_id']

            if long(peer_hdr['ts_secs']) == 0:
                mrt_bgp_msg.TIMESTAMP_SEC = time.time()
            else:
                mrt_bgp_msg.TIMESTAMP_SEC = peer_hdr['ts_secs']

            mrt_bgp_msg.TIMESTAMP_USEC = peer_hdr['ts_usecs']

            mrt_bgp_msg.PEER_KEY = r_hash + '_' + mrt_bgp_msg.PEER_BGP_ID + '_' + str(mrt_bgp_msg.PEER_RD)

            if c_hash in self.COLLECTORS:
                mrt_bgp_msg.COLLECTOR_ADMIN_ID = self.COLLECTORS[c_hash]['admin_id']

            if r_hash in self.ROUTERS:
                mrt_bgp_msg.ROUTER_NAME = self.ROUTERS[r_hash]['name']

            if mrt_bgp_msg.PEER_KEY in self.PEERS:
                mrt_bgp_msg.LOCAL_ADDR = self.PEERS[mrt_bgp_msg.PEER_KEY]['local_ip']
                mrt_bgp_msg.LOCAL_ASN = self.PEERS[mrt_bgp_msg.PEER_KEY]['local_asn']
                mrt_bgp_msg.PEER_ASN_LENGTH = self.PEERS[mrt_bgp_msg.PEER_KEY]['asn_len']

            # queue message
            self._bgp4mp_queue.put(mrt_bgp_msg)

        else:
            self.LOG.debug("BMP message type %r not used" % bmp_hdrs['type'])
<commit_msg>supports latest version of kafka consumer (v 1.3.5)<commit_after># -*- coding: utf-8 -*-
"""OpenBMP MRT

  Copyright (c) 2013-2016 Cisco Systems, Inc. and others.  All rights reserved.
  This program and the accompanying materials are made available under the
  terms of the Eclipse Public License v1.0 which accompanies this distribution,
  and is available at http://www.eclipse.org/legal/epl-v10.html

  .. moduleauthor:: Tim Evens <tievens@cisco.com>
"""
import multiprocessing
import socket
import kafka
import kafka.common
import time

from openbmp.mrt_bgp4mp_object import MrtBgpMpObject
from openbmp.logger import init_mp_logger

from openbmp.api.parsed.message import Message
from openbmp.api.parsed.message import MsgBusFields
from openbmp.api.parsed.message import Collector
from openbmp.api.parsed.message import Router
from openbmp.api.parsed.message import Peer

from openbmp.bmp import bmp_parse_bmphdr, bmp_parse_peerhdr

class BMPConsumer(multiprocessing.Process):
    """ OpenBMP consumer

        Consumes openbmp.parsed.collector, openbmp.parsed.router, and openbmp.bmp_raw
        message feeds from Kafka.  Collector and router messages are cached.
    """
    # Memory cache of collectors
    COLLECTORS = {}

    # Memory cache of routers
    ROUTERS = {}

    # Memory cache of peers
    PEERS = {}

    def __init__(self, cfg, mrt_bgp4mp_queue, log_queue):
        """ Constructor

            :param cfg:               Configuration dictionary
            :param mrt_bgp4mp_queue:  Output for MRT BGP4MP type
            :param log_queue:         Logging queue - sync logging
        """
        multiprocessing.Process.__init__(self)
        self._stop = multiprocessing.Event()

        self._cfg = cfg
        self._bgp4mp_queue = mrt_bgp4mp_queue
        self._log_queue = log_queue
        self.LOG = None

    def run(self):
        """ Override """
        self.LOG = init_mp_logger("bmp_consumer", self._log_queue)

        # Enable to topics/feeds
        topics = ['openbmp.parsed.collector', 'openbmp.parsed.router', 'openbmp.parsed.peer', 'openbmp.bmp_raw']

        self.LOG.info("Running bmp_consumer")

        # wait for config to load
        while not self.stopped():
            if self._cfg and 'kafka' in self._cfg:
                break

        try:
            # connect and bind to topics
            self.LOG.info("Connecting to %s ... takes a minute to load offsets and topics, please wait" % self._cfg['kafka']['servers'])
            consumer = kafka.KafkaConsumer(*topics,
                                           bootstrap_servers=self._cfg['kafka']['servers'],
                                           client_id=self._cfg['kafka']['client_id'] + '-' + socket.gethostname(),
                                           group_id=self._cfg['kafka']['group_id'],
                                           enable_auto_commit=True,
                                           auto_commit_interval_ms=1000,
                                           auto_offset_reset='largest' if
                                                            self._cfg['kafka']['offset_reset_largest'] else "smallest")

            self.LOG.info("Connected, now consuming")

            prev_ts = time.time()

            # Loop till stopped
            while not self.stopped():

                # Read messages
                for m in consumer:
                    if self.stopped():
                        break

                    self.process_msg(m)

        except kafka.common.KafkaUnavailableError as err:
            self.LOG.error("Kafka Error: %s" % str(err))

        except KeyboardInterrupt:
            pass

        self.LOG.info("consumer stopped")

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.is_set()

    def process_msg(self, msg):
        """ Process the message

            :param msg:     Message consumed
        """

        m = Message(msg.value)  # Gets body of kafka message.

        if msg.topic == 'openbmp.parsed.collector':
            self.process_collector_msg(m)

        elif msg.topic == 'openbmp.parsed.router':
            self.process_router_msg(m)

        elif msg.topic == 'openbmp.parsed.peer':
            self.process_peer_msg(m)

        elif msg.topic == 'openbmp.bmp_raw':
            self.process_bmp_raw_msg(m)

    def process_collector_msg(self, msg):
        """ Process collector message and cache info for use later

            :param msg:      Message object
        """
        obj = Collector(msg)

        map_list = obj.getRowMap()

        for row in map_list:
            if len(row):
                try:
                    self.LOG.debug("collector: %s [%s]", row[MsgBusFields.ADMIN_ID.getName()], row[MsgBusFields.ACTION.getName()])

                    if row[MsgBusFields.ACTION.getName()] in ('started', 'heartbeat', 'change'):

                        # Update collector hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.COLLECTORS:
                            self.COLLECTORS[row[MsgBusFields.HASH.getName()]] = {'admin_id': row[MsgBusFields.ADMIN_ID.getName()]}

                    else:
                        self.COLLECTORS.pop(row[MsgBusFields.HASH.getName()], None)

                except:
                    pass

    def process_router_msg(self, msg):
        """ Process Router message

            :param msg:      Message object
        """
        obj = Router(msg)

        map_list = obj.getRowMap()

        # Log messages
        for row in map_list:
            if len(row):
                try:
                    self.LOG.info("router: [%s] %s %s", row[MsgBusFields.ACTION.getName()],
                                  row[MsgBusFields.NAME.getName()], row[MsgBusFields.IP_ADDRESS.getName()])

                    if row[MsgBusFields.ACTION.getName()] in ('first', 'init'):
                        # Update the router hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.ROUTERS:
                            self.ROUTERS[row[MsgBusFields.HASH.getName()]] = {'ip': msg.getRouterIp(), 'name': row[MsgBusFields.NAME.getName()]}
                    # else:
                    #     self.ROUTERS.pop(row[MsgBusFields.HASH.getName()], None)

                except:
                    self.LOG.debug("router parse error")
                    pass

    def process_peer_msg(self, msg):
        """ Process Peer message

            :param msg:      Message object
        """

        obj = Peer(msg)

        map_list = obj.getRowMap()

        # Log messages
        for row in map_list:
            if len(row):
                try:
                    peer_key = row[MsgBusFields.ROUTER_HASH.getName()] + '_' + row[MsgBusFields.REMOTE_BGP_ID.getName()] + '_' + row[MsgBusFields.PEER_RD.getName()]

                    self.LOG.info("peer: [%s] %s %s", row[MsgBusFields.ACTION.getName()],
                                  row[MsgBusFields.NAME.getName()], row[MsgBusFields.REMOTE_IP.getName()]) # Peer IP address

                    if row[MsgBusFields.ACTION.getName()] in ('up'):

                        asn_len = 2
                        if ('4 Octet ASN' in row[MsgBusFields.ADV_CAP.getName()] and
                            '4 Octet ASN' in row[MsgBusFields.RECV_CAP.getName()]):
                            asn_len = 4

                        # Update the peer hash/cache
                        if row[MsgBusFields.HASH.getName()] not in self.PEERS:
                            self.PEERS[peer_key] = {'remote_ip': row[MsgBusFields.REMOTE_IP.getName()],
                                                    'remote_asn': row[MsgBusFields.REMOTE_ASN.getName()],
                                                    'local_ip': row[MsgBusFields.LOCAL_IP.getName()],
                                                    'local_asn': row[MsgBusFields.LOCAL_ASN.getName()],
                                                    'asn_len': asn_len,
                                                    'rd': row[MsgBusFields.PEER_RD.getName()]}
                except:
                    self.LOG.debug("peer parse error")
                    pass

    def process_bmp_raw_msg(self, msg):
        """ Process BMP RAW message

        :param msg:      Message object
        """

        c_hash = msg.getCollector_hash_id() # Collector Hash ID
        r_hash = msg.getRouter_hash_id() # Router Hash ID
        r_ip = msg.getRouterIp() # Router IP address
        data = msg.getContent() # Message data to be consumed (should not contain headers)

        mrt_bgp_msg = MrtBgpMpObject()

        mrt_bgp_msg.COLLECTOR_HASH_ID = c_hash
        mrt_bgp_msg.ROUTER_IP = r_ip

        mrt_bgp_msg.PEER_KEY = ""

        # Parse the BMP header
        bmp_hdrs = bmp_parse_bmphdr(data)

        # Parse the BMP headers
        if bmp_hdrs['type'] in ('ROUTE_MON'):
            peer_hdr = bmp_parse_peerhdr(data[6:])
            mrt_bgp_msg.BMP_DATA = data[48:]

            mrt_bgp_msg.PEER_ADDR = peer_hdr['addr']
            mrt_bgp_msg.PEER_RD = peer_hdr['dist_id']
            mrt_bgp_msg.PEER_ASN = peer_hdr['asn']
            mrt_bgp_msg.PEER_BGP_ID = peer_hdr['bgp_id']

            if long(peer_hdr['ts_secs']) == 0:
                mrt_bgp_msg.TIMESTAMP_SEC = time.time()
            else:
                mrt_bgp_msg.TIMESTAMP_SEC = peer_hdr['ts_secs']

            mrt_bgp_msg.TIMESTAMP_USEC = peer_hdr['ts_usecs']

            mrt_bgp_msg.PEER_KEY = r_hash + '_' + mrt_bgp_msg.PEER_BGP_ID + '_' + str(mrt_bgp_msg.PEER_RD)

            if c_hash in self.COLLECTORS:
                mrt_bgp_msg.COLLECTOR_ADMIN_ID = self.COLLECTORS[c_hash]['admin_id']

            if r_hash in self.ROUTERS:
                mrt_bgp_msg.ROUTER_NAME = self.ROUTERS[r_hash]['name']

            if mrt_bgp_msg.PEER_KEY in self.PEERS:
                mrt_bgp_msg.LOCAL_ADDR = self.PEERS[mrt_bgp_msg.PEER_KEY]['local_ip']
                mrt_bgp_msg.LOCAL_ASN = self.PEERS[mrt_bgp_msg.PEER_KEY]['local_asn']
                mrt_bgp_msg.PEER_ASN_LENGTH = self.PEERS[mrt_bgp_msg.PEER_KEY]['asn_len']

            # queue message
            self._bgp4mp_queue.put(mrt_bgp_msg)

        else:
            self.LOG.debug("BMP message type %r not used" % bmp_hdrs['type'])
<|endoftext|>


<commit_before>#! /usr/bin/env python

"""Check design."""
from __future__ import print_function
import sys
import pandas as pd
import os

class CheckDesign():
    """Check experimental design are tab delimited."""

    def __init__(self, exp):
        """Initialize."""
        self.design_file = exp

    def tab(self):
        """Check if the file is tab delimited and has more than 2 columns."""
        with open(self.design_file, 'r') as dfile:
            header = dfile.readline()
            num_tab = len(header.split("\t"))
            if num_tab > 2:
                line_status = []
                for line in dfile:
                    if len(line.split("\t")) == num_tab:
                        line_status.append(True)
                    else:
                        line_status.append(False)
                if all(line_status) is True:
                    return True
                else:
                    sys.exit('Experimental design file not correctly \
                              formatted!')
            else:
                sys.exit('Experimental design file is not correctly formatted!'
                         )

    def header(self):
        """Check if headers are present."""
        with open(self.design_file, 'r') as dfile:
            header = dfile.readline().rstrip()
            if header == "ID\tFiles\tGroup":
                return True
            else:
                sys.exit('The header of experimental design file must be ID\tFiles\tGroup\n')

    def sample_name(self):
        """Check if sample names are unique."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            sample_names = []
            for line in dfile:
                sample_names.append(line.split("\t")[0])
            if len(sample_names) == len(set(sample_names)):
                return True
            else:
                return False
                sys.exit('Sample names must be unique!')

    def file_name(self):
        """Check if file names are unique."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            file_names = []
            for line in dfile:
                file_names.append(line.split("\t")[1])
            if len(file_names) == len(set(file_names)):
                return True
            else:
                sys.exit('Input files must be unique!')

    def group_name(self):
        """Check if there are at least two groups."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            groups = []
            for line in dfile:
                groups.append(line.split("\t")[2])
            if len(groups) < 2:
                sys.exit('There must be at least two groups for comparisons!')
            else:
                return True

    def fastq_exists(self):
        """Check if specified fastq files are present."""
        fastq_files = self.extract_sample_fastqs()
        for samp, fastqs in fastq_files.iteritems():
            if isinstance(fastqs, str):
                files = fastqs.split(":")
                for f in files:
                    if os.path.exists(f):
                        pass
                    else:
                        err_message = "%s does not exist"
                        sys.exit(err_message % f)
            elif isinstance(fastqs, list):
                for pairs in fastqs:
                    files = pairs.split(";")
                    for f in files:
                        if os.path.exists(f):
                            pass
                        else:
                            err_message = "%s does not exist"
                            sys.exit(err_message % f)
        return True

    def extract_sample_fastqs(self):
        """Extract sample and fastqc from design file."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            sample = []
            sample_fastqs = {}
            for line in dfile:
                sample = line.split("\t")[0]
                if ';' in line.split("\t")[1]:
                    fastq_pairs = line.split("\t")[1].split(":")
                else:
                    fastq_pairs = line.split("\t")[1]
                sample_fastqs[sample] = fastq_pairs
        return sample_fastqs

    def sample_suff(self, method):
        """Check if enough number of samples are present for Deseq analysis."""
        exp_design = pd.read_csv(self.design_file, sep='\t')
        if method in ['both', 'Deseq']:
            group_count = exp_design['Group'].value_counts()
            for e in [1, 2]:
                if e in group_count.values:
                    sys.exit("There must be at least three samples \
                              to run DeSeq")
                    break
                else:
                    pass
        return True
<commit_msg>[skip ci] more descriptive error message<commit_after>#! /usr/bin/env python

"""Check design."""
from __future__ import print_function
import sys
import pandas as pd
import os

class CheckDesign():
    """Check experimental design are tab delimited."""

    def __init__(self, exp):
        """Initialize."""
        self.design_file = exp

    def tab(self):
        """Check if the file is tab delimited and has more than 2 columns."""
        with open(self.design_file, 'r') as dfile:
            header = dfile.readline()
            num_tab = len(header.split("\t"))
            if num_tab > 2:
                line_status = []
                for line in dfile:
                    if len(line.split("\t")) == num_tab:
                        line_status.append(True)
                    else:
                        line_status.append(False)
                if all(line_status) is True:
                    return True
                else:
                    sys.exit('Experimental design file not correctly \
                              formatted!')
            else:
                sys.exit('Experimental design file is not correctly formatted!\n \
                    There is only one or no column!'
                         )

    def header(self):
        """Check if headers are present."""
        with open(self.design_file, 'r') as dfile:
            header = dfile.readline().rstrip()
            if header == "ID\tFiles\tGroup":
                return True
            else:
                sys.exit('The header of experimental design file must be ID\tFiles\tGroup\n')

    def sample_name(self):
        """Check if sample names are unique."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            sample_names = []
            for line in dfile:
                sample_names.append(line.split("\t")[0])
            if len(sample_names) == len(set(sample_names)):
                return True
            else:
                return False
                sys.exit('Sample names must be unique!')

    def file_name(self):
        """Check if file names are unique."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            file_names = []
            for line in dfile:
                file_names.append(line.split("\t")[1])
            if len(file_names) == len(set(file_names)):
                return True
            else:
                sys.exit('Input files must be unique!')

    def group_name(self):
        """Check if there are at least two groups."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            groups = []
            for line in dfile:
                groups.append(line.split("\t")[2])
            if len(groups) < 2:
                sys.exit('There must be at least two groups for comparisons!')
            else:
                return True

    def fastq_exists(self):
        """Check if specified fastq files are present."""
        fastq_files = self.extract_sample_fastqs()
        for samp, fastqs in fastq_files.iteritems():
            if isinstance(fastqs, str):
                files = fastqs.split(":")
                for f in files:
                    if os.path.exists(f):
                        pass
                    else:
                        err_message = "%s does not exist"
                        sys.exit(err_message % f)
            elif isinstance(fastqs, list):
                for pairs in fastqs:
                    files = pairs.split(";")
                    for f in files:
                        if os.path.exists(f):
                            pass
                        else:
                            err_message = "%s does not exist"
                            sys.exit(err_message % f)
        return True

    def extract_sample_fastqs(self):
        """Extract sample and fastqc from design file."""
        with open(self.design_file, 'r') as dfile:
            next(dfile)
            sample = []
            sample_fastqs = {}
            for line in dfile:
                sample = line.split("\t")[0]
                if ';' in line.split("\t")[1]:
                    fastq_pairs = line.split("\t")[1].split(":")
                else:
                    fastq_pairs = line.split("\t")[1]
                sample_fastqs[sample] = fastq_pairs
        return sample_fastqs

    def sample_suff(self, method):
        """Check if enough number of samples are present for Deseq analysis."""
        exp_design = pd.read_csv(self.design_file, sep='\t')
        if method in ['both', 'Deseq']:
            group_count = exp_design['Group'].value_counts()
            for e in [1, 2]:
                if e in group_count.values:
                    sys.exit("There must be at least three samples \
                              to run DeSeq")
                    break
                else:
                    pass
        return True
<|endoftext|>


<commit_before>import re
import os
import sys
import warnings

from numpy.distutils.cpuinfo import cpu
from numpy.distutils.fcompiler import FCompiler
from numpy.distutils.exec_command import exec_command
from numpy.distutils.misc_util import msvc_runtime_library

compilers = ['GnuFCompiler', 'Gnu95FCompiler']

TARGET_R = re.compile("Target: ([a-zA-Z0-9_\-]*)")
class GnuFCompiler(FCompiler):
    compiler_type = 'gnu'
    compiler_aliases = ('g77',)
    description = 'GNU Fortran 77 compiler'

    def gnu_version_match(self, version_string):
        """Handle the different versions of GNU fortran compilers"""
        m = re.match(r'GNU Fortran', version_string)
        if not m:
            return None
        m = re.match(r'GNU Fortran\s+95.*?([0-9-.]+)', version_string)
        if m:
            return ('gfortran', m.group(1))
        m = re.match(r'GNU Fortran.*?([0-9-.]+)', version_string)
        if m:
            v = m.group(1)
            if v.startswith('0') or v.startswith('2') or v.startswith('3'):
                # the '0' is for early g77's
                return ('g77', v)
            else:
                # at some point in the 4.x series, the ' 95' was dropped
                # from the version string
                return ('gfortran', v)

    def version_match(self, version_string):
        v = self.gnu_version_match(version_string)
        if not v or v[0] != 'g77':
            return None
        return v[1]

    # 'g77 --version' results
    # SunOS: GNU Fortran (GCC 3.2) 3.2 20020814 (release)
    # Debian: GNU Fortran (GCC) 3.3.3 20040110 (prerelease) (Debian)
    #         GNU Fortran (GCC) 3.3.3 (Debian 20040401)
    #         GNU Fortran 0.5.25 20010319 (prerelease)
    # Redhat: GNU Fortran (GCC 3.2.2 20030222 (Red Hat Linux 3.2.2-5)) 3.2.2 20030222 (Red Hat Linux 3.2.2-5)
    # GNU Fortran (GCC) 3.4.2 (mingw-special)

    possible_executables = ['g77', 'f77']
    executables = {
        'version_cmd'  : [None, "--version"],
        'compiler_f77' : [None, "-g", "-Wall", "-fno-second-underscore"],
        'compiler_f90' : None, # Use --fcompiler=gnu95 for f90 codes
        'compiler_fix' : None,
        'linker_so'    : [None, "-g", "-Wall"],
        'archiver'     : ["ar", "-cr"],
        'ranlib'       : ["ranlib"],
        'linker_exe'   : [None, "-g", "-Wall"]
        }
    module_dir_switch = None
    module_include_switch = None

    # Cygwin: f771: warning: -fPIC ignored for target (all code is
    # position independent)
    if os.name != 'nt' and sys.platform != 'cygwin':
        pic_flags = ['-fPIC']

    # use -mno-cygwin for g77 when Python is not Cygwin-Python
    if sys.platform == 'win32':
        for key in ['version_cmd', 'compiler_f77', 'linker_so', 'linker_exe']:
            executables[key].append('-mno-cygwin')

    g2c = 'g2c'

    suggested_f90_compiler = 'gnu95'

    #def get_linker_so(self):
    #    # win32 linking should be handled by standard linker
    #    # Darwin g77 cannot be used as a linker.
    #    #if re.match(r'(darwin)', sys.platform):
    #    #    return
    #    return FCompiler.get_linker_so(self)

    def get_flags_linker_so(self):
        opt = self.linker_so[1:]
        if sys.platform=='darwin':
            # MACOSX_DEPLOYMENT_TARGET must be at least 10.3. This is
            # a reasonable default value even when building on 10.4 when using
            # the official Python distribution and those derived from it (when
            # not broken).
            target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
            if target is None or target == '':
                target = '10.3'
            major, minor = target.split('.')
            if int(minor) < 3:
                minor = '3'
                warnings.warn('Environment variable '
                    'MACOSX_DEPLOYMENT_TARGET reset to %s.%s' % (major, minor))
            os.environ['MACOSX_DEPLOYMENT_TARGET'] = '%s.%s' % (major,
                minor)

            opt.extend(['-undefined', 'dynamic_lookup', '-bundle'])
        else:
            opt.append("-shared")
        if sys.platform.startswith('sunos'):
            # SunOS often has dynamically loaded symbols defined in the
            # static library libg2c.a  The linker doesn't like this.  To
            # ignore the problem, use the -mimpure-text flag.  It isn't
            # the safest thing, but seems to work. 'man gcc' says:
            # ".. Instead of using -mimpure-text, you should compile all
            #  source code with -fpic or -fPIC."
            opt.append('-mimpure-text')
        return opt

    def get_libgcc_dir(self):
        status, output = exec_command(self.compiler_f77 +
                                      ['-print-libgcc-file-name'],
                                      use_tee=0)
        if not status:
            return os.path.dirname(output)
        return None

    def get_library_dirs(self):
        opt = []
        if sys.platform[:5] != 'linux':
            d = self.get_libgcc_dir()
            if d:
                # if windows and not cygwin, libg2c lies in a different folder
                if sys.platform == 'win32' and not d.startswith('/usr/lib'):
                    d = os.path.normpath(d)
                    if not os.path.exists(os.path.join(d, "lib%s.a" % self.g2c)):
                        d2 = os.path.abspath(os.path.join(d,
                                                          '../../../../lib'))
                        if os.path.exists(os.path.join(d2, "lib%s.a" % self.g2c)):
                            opt.append(d2)
                opt.append(d)
        return opt

    def get_libraries(self):
        opt = []
        d = self.get_libgcc_dir()
        if d is not None:
            g2c = self.g2c + '-pic'
            f = self.static_lib_format % (g2c, self.static_lib_extension)
            if not os.path.isfile(os.path.join(d,f)):
                g2c = self.g2c
        else:
            g2c = self.g2c

        if g2c is not None:
            opt.append(g2c)
        c_compiler = self.c_compiler
        if sys.platform == 'win32' and c_compiler and \
               c_compiler.compiler_type=='msvc':
            # the following code is not needed (read: breaks) when using MinGW
            # in case want to link F77 compiled code with MSVC
            opt.append('gcc')
            runtime_lib = msvc_runtime_library()
            if runtime_lib:
                opt.append(runtime_lib)
        if sys.platform == 'darwin':
            opt.append('cc_dynamic')
        return opt

    def get_flags_debug(self):
        return ['-g']

    def get_flags_opt(self):
        if self.get_version()<='3.3.3':
            # With this compiler version building Fortran BLAS/LAPACK
            # with -O3 caused failures in lib.lapack heevr,syevr tests.
            opt = ['-O2']
        else:
            opt = ['-O3']
        opt.append('-funroll-loops')
        return opt

    def get_flags_arch(self):
        return []

class Gnu95FCompiler(GnuFCompiler):
    compiler_type = 'gnu95'
    compiler_aliases = ('gfortran',)
    description = 'GNU Fortran 95 compiler'

    def version_match(self, version_string):
        v = self.gnu_version_match(version_string)
        if not v or v[0] != 'gfortran':
            return None
        return v[1]

    # 'gfortran --version' results:
    # XXX is the below right?
    # Debian: GNU Fortran 95 (GCC 4.0.3 20051023 (prerelease) (Debian 4.0.2-3))
    #         GNU Fortran 95 (GCC) 4.1.2 20061115 (prerelease) (Debian 4.1.1-21)
    # OS X: GNU Fortran 95 (GCC) 4.1.0
    #       GNU Fortran 95 (GCC) 4.2.0 20060218 (experimental)
    #       GNU Fortran (GCC) 4.3.0 20070316 (experimental)

    possible_executables = ['gfortran', 'f95']
    executables = {
        'version_cmd'  : ["<F90>", "--version"],
        'compiler_f77' : [None, "-Wall", "-ffixed-form",
                          "-fno-second-underscore"],
        'compiler_f90' : [None, "-Wall", "-fno-second-underscore"],
        'compiler_fix' : [None, "-Wall", "-ffixed-form",
                          "-fno-second-underscore"],
        'linker_so'    : ["<F90>", "-Wall"],
        'archiver'     : ["ar", "-cr"],
        'ranlib'       : ["ranlib"],
        'linker_exe'   : [None, "-Wall"]
        }

    # use -mno-cygwin flag for g77 when Python is not Cygwin-Python
    if sys.platform == 'win32':
        for key in ['version_cmd', 'compiler_f77', 'compiler_f90',
                    'compiler_fix', 'linker_so', 'linker_exe']:
            executables[key].append('-mno-cygwin')

    module_dir_switch = '-J'
    module_include_switch = '-I'

    g2c = 'gfortran'

    # Note that this is here instead of GnuFCompiler as gcc < 4 uses a
    # different output format (which isn't as useful) than gcc >= 4,
    # and we don't have to worry about g77 being universal (as it can't be).
    def target_architecture(self, extra_opts=()):
        """Return the architecture that the compiler will build for.
        This is most useful for detecting universal compilers in OS X."""
        extra_opts = list(extra_opts)
        status, output = exec_command(self.compiler_f90 + ['-v'] + extra_opts,
                                      use_tee=False)
        if status == 0:
            m = re.match(r'(?m)^Target: (.*)$', output)
            if m:
                return m.group(1)
        return None

    def is_universal_compiler(self):
        """Return True if this compiler can compile universal binaries
        (for OS X).

        Currently only checks for i686 and powerpc architectures (no 64-bit
        support yet).
        """
        if sys.platform != 'darwin':
            return False
        i686_arch = self.target_architecture(extra_opts=['-arch', 'i686'])
        if not i686_arch or not i686_arch.startswith('i686-'):
            return False
        ppc_arch = self.target_architecture(extra_opts=['-arch', 'ppc'])
        if not ppc_arch or not ppc_arch.startswith('powerpc-'):
            return False
        return True

    def _add_arches_for_universal_build(self, flags):
        if self.is_universal_compiler():
            flags[:0] = ['-arch', 'i686', '-arch', 'ppc']
        return flags

    def get_flags(self):
        flags = GnuFCompiler.get_flags(self)
        return self._add_arches_for_universal_build(flags)

    def get_flags_linker_so(self):
        flags = GnuFCompiler.get_flags_linker_so(self)
        return self._add_arches_for_universal_build(flags)

    def get_libraries(self):
        opt = GnuFCompiler.get_libraries(self)
        if sys.platform == 'darwin':
            opt.remove('cc_dynamic')
        return opt

    def get_target(self):
        status, output = exec_command(self.compiler_f77 +
                                      ['-v'],
                                      use_tee=0)
        if not status:
	    m = TARGET_R.search(output)
	    if m:
	        print m.group(1)
        return ""

if __name__ == '__main__':
    from distutils import log
    log.set_verbosity(2)
    compiler = GnuFCompiler()
    compiler.customize()
    print compiler.get_version()
    raw_input('Press ENTER to continue...')
    try:
        compiler = Gnu95FCompiler()
        compiler.customize()
        print compiler.get_version()
    except Exception, msg:
        print msg
    raw_input('Press ENTER to continue...')
<commit_msg>Fix get_target.<commit_after>import re
import os
import sys
import warnings

from numpy.distutils.cpuinfo import cpu
from numpy.distutils.fcompiler import FCompiler
from numpy.distutils.exec_command import exec_command
from numpy.distutils.misc_util import msvc_runtime_library

compilers = ['GnuFCompiler', 'Gnu95FCompiler']

TARGET_R = re.compile("Target: ([a-zA-Z0-9_\-]*)")
class GnuFCompiler(FCompiler):
    compiler_type = 'gnu'
    compiler_aliases = ('g77',)
    description = 'GNU Fortran 77 compiler'

    def gnu_version_match(self, version_string):
        """Handle the different versions of GNU fortran compilers"""
        m = re.match(r'GNU Fortran', version_string)
        if not m:
            return None
        m = re.match(r'GNU Fortran\s+95.*?([0-9-.]+)', version_string)
        if m:
            return ('gfortran', m.group(1))
        m = re.match(r'GNU Fortran.*?([0-9-.]+)', version_string)
        if m:
            v = m.group(1)
            if v.startswith('0') or v.startswith('2') or v.startswith('3'):
                # the '0' is for early g77's
                return ('g77', v)
            else:
                # at some point in the 4.x series, the ' 95' was dropped
                # from the version string
                return ('gfortran', v)

    def version_match(self, version_string):
        v = self.gnu_version_match(version_string)
        if not v or v[0] != 'g77':
            return None
        return v[1]

    # 'g77 --version' results
    # SunOS: GNU Fortran (GCC 3.2) 3.2 20020814 (release)
    # Debian: GNU Fortran (GCC) 3.3.3 20040110 (prerelease) (Debian)
    #         GNU Fortran (GCC) 3.3.3 (Debian 20040401)
    #         GNU Fortran 0.5.25 20010319 (prerelease)
    # Redhat: GNU Fortran (GCC 3.2.2 20030222 (Red Hat Linux 3.2.2-5)) 3.2.2 20030222 (Red Hat Linux 3.2.2-5)
    # GNU Fortran (GCC) 3.4.2 (mingw-special)

    possible_executables = ['g77', 'f77']
    executables = {
        'version_cmd'  : [None, "--version"],
        'compiler_f77' : [None, "-g", "-Wall", "-fno-second-underscore"],
        'compiler_f90' : None, # Use --fcompiler=gnu95 for f90 codes
        'compiler_fix' : None,
        'linker_so'    : [None, "-g", "-Wall"],
        'archiver'     : ["ar", "-cr"],
        'ranlib'       : ["ranlib"],
        'linker_exe'   : [None, "-g", "-Wall"]
        }
    module_dir_switch = None
    module_include_switch = None

    # Cygwin: f771: warning: -fPIC ignored for target (all code is
    # position independent)
    if os.name != 'nt' and sys.platform != 'cygwin':
        pic_flags = ['-fPIC']

    # use -mno-cygwin for g77 when Python is not Cygwin-Python
    if sys.platform == 'win32':
        for key in ['version_cmd', 'compiler_f77', 'linker_so', 'linker_exe']:
            executables[key].append('-mno-cygwin')

    g2c = 'g2c'

    suggested_f90_compiler = 'gnu95'

    #def get_linker_so(self):
    #    # win32 linking should be handled by standard linker
    #    # Darwin g77 cannot be used as a linker.
    #    #if re.match(r'(darwin)', sys.platform):
    #    #    return
    #    return FCompiler.get_linker_so(self)

    def get_flags_linker_so(self):
        opt = self.linker_so[1:]
        if sys.platform=='darwin':
            # MACOSX_DEPLOYMENT_TARGET must be at least 10.3. This is
            # a reasonable default value even when building on 10.4 when using
            # the official Python distribution and those derived from it (when
            # not broken).
            target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
            if target is None or target == '':
                target = '10.3'
            major, minor = target.split('.')
            if int(minor) < 3:
                minor = '3'
                warnings.warn('Environment variable '
                    'MACOSX_DEPLOYMENT_TARGET reset to %s.%s' % (major, minor))
            os.environ['MACOSX_DEPLOYMENT_TARGET'] = '%s.%s' % (major,
                minor)

            opt.extend(['-undefined', 'dynamic_lookup', '-bundle'])
        else:
            opt.append("-shared")
        if sys.platform.startswith('sunos'):
            # SunOS often has dynamically loaded symbols defined in the
            # static library libg2c.a  The linker doesn't like this.  To
            # ignore the problem, use the -mimpure-text flag.  It isn't
            # the safest thing, but seems to work. 'man gcc' says:
            # ".. Instead of using -mimpure-text, you should compile all
            #  source code with -fpic or -fPIC."
            opt.append('-mimpure-text')
        return opt

    def get_libgcc_dir(self):
        status, output = exec_command(self.compiler_f77 +
                                      ['-print-libgcc-file-name'],
                                      use_tee=0)
        if not status:
            return os.path.dirname(output)
        return None

    def get_library_dirs(self):
        opt = []
        if sys.platform[:5] != 'linux':
            d = self.get_libgcc_dir()
            if d:
                # if windows and not cygwin, libg2c lies in a different folder
                if sys.platform == 'win32' and not d.startswith('/usr/lib'):
                    d = os.path.normpath(d)
                    if not os.path.exists(os.path.join(d, "lib%s.a" % self.g2c)):
                        d2 = os.path.abspath(os.path.join(d,
                                                          '../../../../lib'))
                        if os.path.exists(os.path.join(d2, "lib%s.a" % self.g2c)):
                            opt.append(d2)
                opt.append(d)
        return opt

    def get_libraries(self):
        opt = []
        d = self.get_libgcc_dir()
        if d is not None:
            g2c = self.g2c + '-pic'
            f = self.static_lib_format % (g2c, self.static_lib_extension)
            if not os.path.isfile(os.path.join(d,f)):
                g2c = self.g2c
        else:
            g2c = self.g2c

        if g2c is not None:
            opt.append(g2c)
        c_compiler = self.c_compiler
        if sys.platform == 'win32' and c_compiler and \
               c_compiler.compiler_type=='msvc':
            # the following code is not needed (read: breaks) when using MinGW
            # in case want to link F77 compiled code with MSVC
            opt.append('gcc')
            runtime_lib = msvc_runtime_library()
            if runtime_lib:
                opt.append(runtime_lib)
        if sys.platform == 'darwin':
            opt.append('cc_dynamic')
        return opt

    def get_flags_debug(self):
        return ['-g']

    def get_flags_opt(self):
        if self.get_version()<='3.3.3':
            # With this compiler version building Fortran BLAS/LAPACK
            # with -O3 caused failures in lib.lapack heevr,syevr tests.
            opt = ['-O2']
        else:
            opt = ['-O3']
        opt.append('-funroll-loops')
        return opt

    def get_flags_arch(self):
        return []

class Gnu95FCompiler(GnuFCompiler):
    compiler_type = 'gnu95'
    compiler_aliases = ('gfortran',)
    description = 'GNU Fortran 95 compiler'

    def version_match(self, version_string):
        v = self.gnu_version_match(version_string)
        if not v or v[0] != 'gfortran':
            return None
        return v[1]

    # 'gfortran --version' results:
    # XXX is the below right?
    # Debian: GNU Fortran 95 (GCC 4.0.3 20051023 (prerelease) (Debian 4.0.2-3))
    #         GNU Fortran 95 (GCC) 4.1.2 20061115 (prerelease) (Debian 4.1.1-21)
    # OS X: GNU Fortran 95 (GCC) 4.1.0
    #       GNU Fortran 95 (GCC) 4.2.0 20060218 (experimental)
    #       GNU Fortran (GCC) 4.3.0 20070316 (experimental)

    possible_executables = ['gfortran', 'f95']
    executables = {
        'version_cmd'  : ["<F90>", "--version"],
        'compiler_f77' : [None, "-Wall", "-ffixed-form",
                          "-fno-second-underscore"],
        'compiler_f90' : [None, "-Wall", "-fno-second-underscore"],
        'compiler_fix' : [None, "-Wall", "-ffixed-form",
                          "-fno-second-underscore"],
        'linker_so'    : ["<F90>", "-Wall"],
        'archiver'     : ["ar", "-cr"],
        'ranlib'       : ["ranlib"],
        'linker_exe'   : [None, "-Wall"]
        }

    # use -mno-cygwin flag for g77 when Python is not Cygwin-Python
    if sys.platform == 'win32':
        for key in ['version_cmd', 'compiler_f77', 'compiler_f90',
                    'compiler_fix', 'linker_so', 'linker_exe']:
            executables[key].append('-mno-cygwin')

    module_dir_switch = '-J'
    module_include_switch = '-I'

    g2c = 'gfortran'

    # Note that this is here instead of GnuFCompiler as gcc < 4 uses a
    # different output format (which isn't as useful) than gcc >= 4,
    # and we don't have to worry about g77 being universal (as it can't be).
    def target_architecture(self, extra_opts=()):
        """Return the architecture that the compiler will build for.
        This is most useful for detecting universal compilers in OS X."""
        extra_opts = list(extra_opts)
        status, output = exec_command(self.compiler_f90 + ['-v'] + extra_opts,
                                      use_tee=False)
        if status == 0:
            m = re.match(r'(?m)^Target: (.*)$', output)
            if m:
                return m.group(1)
        return None

    def is_universal_compiler(self):
        """Return True if this compiler can compile universal binaries
        (for OS X).

        Currently only checks for i686 and powerpc architectures (no 64-bit
        support yet).
        """
        if sys.platform != 'darwin':
            return False
        i686_arch = self.target_architecture(extra_opts=['-arch', 'i686'])
        if not i686_arch or not i686_arch.startswith('i686-'):
            return False
        ppc_arch = self.target_architecture(extra_opts=['-arch', 'ppc'])
        if not ppc_arch or not ppc_arch.startswith('powerpc-'):
            return False
        return True

    def _add_arches_for_universal_build(self, flags):
        if self.is_universal_compiler():
            flags[:0] = ['-arch', 'i686', '-arch', 'ppc']
        return flags

    def get_flags(self):
        flags = GnuFCompiler.get_flags(self)
        return self._add_arches_for_universal_build(flags)

    def get_flags_linker_so(self):
        flags = GnuFCompiler.get_flags_linker_so(self)
        return self._add_arches_for_universal_build(flags)

    def get_libraries(self):
        opt = GnuFCompiler.get_libraries(self)
        if sys.platform == 'darwin':
            opt.remove('cc_dynamic')
        return opt

    def get_target(self):
        status, output = exec_command(self.compiler_f77 +
                                      ['-v'],
                                      use_tee=0)
        if not status:
	    m = TARGET_R.search(output)
	    if m:
	        return m.group(1)
        return ""

if __name__ == '__main__':
    from distutils import log
    log.set_verbosity(2)
    compiler = GnuFCompiler()
    compiler.customize()
    print compiler.get_version()
    raw_input('Press ENTER to continue...')
    try:
        compiler = Gnu95FCompiler()
        compiler.customize()
        print compiler.get_version()
    except Exception, msg:
        print msg
    raw_input('Press ENTER to continue...')
<|endoftext|>


<commit_before># Backward compatible module for RandomArray

__all__ = ['ArgumentError','F','beta','binomial','chi_square', 'exponential',
           'gamma', 'get_seed', 'mean_var_test', 'multinomial',
           'multivariate_normal', 'negative_binomial', 'noncentral_F',
           'noncentral_chi_square', 'normal', 'permutation', 'poisson',
           'randint', 'random', 'random_integers', 'seed', 'standard_normal',
           'uniform']

ArgumentError = ValueError

import numpy.random.mtrand as mt
import numpy as Numeric

def seed(x=0, y=0):
    if (x == 0 or y == 0):
        mt.seed()
    else:
        mt.seed((x,y))

def get_seed():
    raise NotImplementedError, \
          "If you want to save the state of the random number generator.\n"\
          "Then you should use obj = numpy.random.get_state() followed by.\n"\
          "numpy.random.set_state(obj)."

def random(shape=[]):
    "random(n) or random([n, m, ...]) returns array of random numbers"
    if shape == []:
        shape = None
    return mt.random_sample(shape)

def uniform(minimum, maximum, shape=[]):
    """uniform(minimum, maximum, shape=[]) returns array of given shape of random reals
    in given range"""
    if shape == []:
        shape = None
    return mt.uniform(minimum, maximum, shape)

def randint(minimum, maximum=None, shape=[]):
    """randint(min, max, shape=[]) = random integers >=min, < max
    If max not given, random integers >= 0, <min"""
    if not isinstance(minimum, int):
        raise ArgumentError, "randint requires first argument integer"
    if maximum is None:
        maximum = minimum
        minimum = 0
    if not isinstance(maximum, int):
        raise ArgumentError, "randint requires second argument integer"
    a = ((maximum-minimum)* random(shape))
    if isinstance(a, Numeric.ndarray):
        return minimum + a.astype(Numeric.int)
    else:
        return minimum + int(a)

def random_integers(maximum, minimum=1, shape=[]):
    """random_integers(max, min=1, shape=[]) = random integers in range min-max inclusive"""
    return randint(minimum, maximum+1, shape)

def permutation(n):
    "permutation(n) = a permutation of indices range(n)"
    return mt.permutation(n)

def standard_normal(shape=[]):
    """standard_normal(n) or standard_normal([n, m, ...]) returns array of
           random numbers normally distributed with mean 0 and standard
           deviation 1"""
    if shape == []:
        shape = None
    return mt.standard_normal(shape)

def normal(mean, std, shape=[]):
    """normal(mean, std, n) or normal(mean, std, [n, m, ...]) returns
    array of random numbers randomly distributed with specified mean and
    standard deviation"""
    if shape == []:
        shape = None
    return mt.normal(mean, std, shape)

def multivariate_normal(mean, cov, shape=[]):
    """multivariate_normal(mean, cov) or multivariate_normal(mean, cov, [m, n, ...])
    returns an array containing multivariate normally distributed random numbers
    with specified mean and covariance.

    mean must be a 1 dimensional array. cov must be a square two dimensional
    array with the same number of rows and columns as mean has elements.

    The first form returns a single 1-D array containing a multivariate
    normal.

    The second form returns an array of shape (m, n, ..., cov.shape[0]).
    In this case, output[i,j,...,:] is a 1-D array containing a multivariate
    normal."""
    if shape == []:
        shape = None
    return mt.multivariate_normal(mean, cov, shape)

def exponential(mean, shape=[]):
    """exponential(mean, n) or exponential(mean, [n, m, ...]) returns array
      of random numbers exponentially distributed with specified mean"""
    if shape == []:
        shape = None
    return mt.exponential(mean, shape)

def beta(a, b, shape=[]):
    """beta(a, b) or beta(a, b, [n, m, ...]) returns array of beta distributed random numbers."""
    if shape == []:
        shape = None
    return mt.beta(a, b, shape)

def gamma(a, r, shape=[]):
    """gamma(a, r) or gamma(a, r, [n, m, ...]) returns array of gamma distributed random numbers."""
    if shape == []:
        shape = None
    return mt.gamma(a, r, shape)

def F(dfn, dfd, shape=[]):
    """F(dfn, dfd) or F(dfn, dfd, [n, m, ...]) returns array of F distributed random numbers with dfn degrees of freedom in the numerator and dfd degrees of freedom in the denominator."""
    if shape == []:
        shape == None
    return mt.f(dfn, dfd, shape)

def noncentral_F(dfn, dfd, nconc, shape=[]):
    """noncentral_F(dfn, dfd, nonc) or noncentral_F(dfn, dfd, nonc, [n, m, ...]) returns array of noncentral F distributed random numbers with dfn degrees of freedom in the numerator and dfd degrees of freedom in the denominator, and noncentrality parameter nconc."""
    if shape == []:
        shape = None
    return mt.noncentral_f(dfn, dfd, nconc, shape)

def chi_square(df, shape=[]):
    """chi_square(df) or chi_square(df, [n, m, ...]) returns array of chi squared distributed random numbers with df degrees of freedom."""
    if shape == []:
        shape = None
    return mt.chisquare(df, shape)

def noncentral_chi_square(df, nconc, shape=[]):
    """noncentral_chi_square(df, nconc) or chi_square(df, nconc, [n, m, ...]) returns array of noncentral chi squared distributed random numbers with df degrees of freedom and noncentrality parameter."""
    if shape == []:
        shape = None
    return mt.noncentral_chisquare(df, nconc, shape)

def binomial(trials, p, shape=[]):
    """binomial(trials, p) or binomial(trials, p, [n, m, ...]) returns array of binomially distributed random integers.

           trials is the number of trials in the binomial distribution.
           p is the probability of an event in each trial of the binomial distribution."""
    if shape == []:
        shape = None
    return mt.binomial(trials, p, shape)

def negative_binomial(trials, p, shape=[]):
    """negative_binomial(trials, p) or negative_binomial(trials, p, [n, m, ...]) returns
           array of negative binomially distributed random integers.

           trials is the number of trials in the negative binomial distribution.
           p is the probability of an event in each trial of the negative binomial distribution."""
    if shape == []:
        shape = None
    return mt.negative_binomial(trials, p, shape)

def multinomial(trials, probs, shape=[]):
    """multinomial(trials, probs) or multinomial(trials, probs, [n, m, ...]) returns
           array of multinomial distributed integer vectors.

           trials is the number of trials in each multinomial distribution.
           probs is a one dimensional array. There are len(prob)+1 events.
           prob[i] is the probability of the i-th event, 0<=i<len(prob).
           The probability of event len(prob) is 1.-Numeric.sum(prob).

       The first form returns a single 1-D array containing one multinomially
           distributed vector.

           The second form returns an array of shape (m, n, ..., len(probs)).
           In this case, output[i,j,...,:] is a 1-D array containing a multinomially
           distributed integer 1-D array."""
    if shape == []:
        shape = None
    return mt.multinomial(trials, probs, shape)

def poisson(mean, shape=[]):
    """poisson(mean) or poisson(mean, [n, m, ...]) returns array of poisson
           distributed random integers with specified mean."""
    if shape == []:
        shape = None
    return mt.poisson(mean, shape)

def mean_var_test(x, type, mean, var, skew=[]):
    n = len(x) * 1.0
    x_mean = Numeric.sum(x,axis=0)/n
    x_minus_mean = x - x_mean
    x_var = Numeric.sum(x_minus_mean*x_minus_mean,axis=0)/(n-1.0)
    print "\nAverage of ", len(x), type
    print "(should be about ", mean, "):", x_mean
    print "Variance of those random numbers (should be about ", var, "):", x_var
    if skew != []:
        x_skew = (Numeric.sum(x_minus_mean*x_minus_mean*x_minus_mean,axis=0)/9998.)/x_var**(3./2.)
        print "Skewness of those random numbers (should be about ", skew, "):", x_skew

def test():
    obj = mt.get_state()
    mt.set_state(obj)
    obj2 = mt.get_state()
    if (obj2[1] - obj[1]).any():
        raise SystemExit, "Failed seed test."
    print "First random number is", random()
    print "Average of 10000 random numbers is", Numeric.sum(random(10000),axis=0)/10000.
    x = random([10,1000])
    if len(x.shape) != 2 or x.shape[0] != 10 or x.shape[1] != 1000:
        raise SystemExit, "random returned wrong shape"
    x.shape = (10000,)
    print "Average of 100 by 100 random numbers is", Numeric.sum(x,axis=0)/10000.
    y = uniform(0.5,0.6, (1000,10))
    if len(y.shape) !=2 or y.shape[0] != 1000 or y.shape[1] != 10:
        raise SystemExit, "uniform returned wrong shape"
    y.shape = (10000,)
    if Numeric.minimum.reduce(y) <= 0.5 or Numeric.maximum.reduce(y) >= 0.6:
        raise SystemExit, "uniform returned out of desired range"
    print "randint(1, 10, shape=[50])"
    print randint(1, 10, shape=[50])
    print "permutation(10)", permutation(10)
    print "randint(3,9)", randint(3,9)
    print "random_integers(10, shape=[20])"
    print random_integers(10, shape=[20])
    s = 3.0
    x = normal(2.0, s, [10, 1000])
    if len(x.shape) != 2 or x.shape[0] != 10 or x.shape[1] != 1000:
        raise SystemExit, "standard_normal returned wrong shape"
    x.shape = (10000,)
    mean_var_test(x, "normally distributed numbers with mean 2 and variance %f"%(s**2,), 2, s**2, 0)
    x = exponential(3, 10000)
    mean_var_test(x, "random numbers exponentially distributed with mean %f"%(s,), s, s**2, 2)
    x = multivariate_normal(Numeric.array([10,20]), Numeric.array(([1,2],[2,4])))
    print "\nA multivariate normal", x
    if x.shape != (2,): raise SystemExit, "multivariate_normal returned wrong shape"
    x = multivariate_normal(Numeric.array([10,20]), Numeric.array([[1,2],[2,4]]), [4,3])
    print "A 4x3x2 array containing multivariate normals"
    print x
    if x.shape != (4,3,2): raise SystemExit, "multivariate_normal returned wrong shape"
    x = multivariate_normal(Numeric.array([-100,0,100]), Numeric.array([[3,2,1],[2,2,1],[1,1,1]]), 10000)
    x_mean = Numeric.sum(x,axis=0)/10000.
    print "Average of 10000 multivariate normals with mean [-100,0,100]"
    print x_mean
    x_minus_mean = x - x_mean
    print "Estimated covariance of 10000 multivariate normals with covariance [[3,2,1],[2,2,1],[1,1,1]]"
    print Numeric.dot(Numeric.transpose(x_minus_mean),x_minus_mean)/9999.
    x = beta(5.0, 10.0, 10000)
    mean_var_test(x, "beta(5.,10.) random numbers", 0.333, 0.014)
    x = gamma(.01, 2., 10000)
    mean_var_test(x, "gamma(.01,2.) random numbers", 2*100, 2*100*100)
    x = chi_square(11., 10000)
    mean_var_test(x, "chi squared random numbers with 11 degrees of freedom", 11, 22, 2*Numeric.sqrt(2./11.))
    x = F(5., 10., 10000)
    mean_var_test(x, "F random numbers with 5 and 10 degrees of freedom", 1.25, 1.35)
    x = poisson(50., 10000)
    mean_var_test(x, "poisson random numbers with mean 50", 50, 50, 0.14)
    print "\nEach element is the result of 16 binomial trials with probability 0.5:"
    print binomial(16, 0.5, 16)
    print "\nEach element is the result of 16 negative binomial trials with probability 0.5:"
    print negative_binomial(16, 0.5, [16,])
    print "\nEach row is the result of 16 multinomial trials with probabilities [0.1, 0.5, 0.1 0.3]:"
    x = multinomial(16, [0.1, 0.5, 0.1], 8)
    print x
    print "Mean = ", Numeric.sum(x,axis=0)/8.

if __name__ == '__main__':
    test()
<commit_msg>fixed typo<commit_after># Backward compatible module for RandomArray

__all__ = ['ArgumentError','F','beta','binomial','chi_square', 'exponential',
           'gamma', 'get_seed', 'mean_var_test', 'multinomial',
           'multivariate_normal', 'negative_binomial', 'noncentral_F',
           'noncentral_chi_square', 'normal', 'permutation', 'poisson',
           'randint', 'random', 'random_integers', 'seed', 'standard_normal',
           'uniform']

ArgumentError = ValueError

import numpy.random.mtrand as mt
import numpy as Numeric

def seed(x=0, y=0):
    if (x == 0 or y == 0):
        mt.seed()
    else:
        mt.seed((x,y))

def get_seed():
    raise NotImplementedError, \
          "If you want to save the state of the random number generator.\n"\
          "Then you should use obj = numpy.random.get_state() followed by.\n"\
          "numpy.random.set_state(obj)."

def random(shape=[]):
    "random(n) or random([n, m, ...]) returns array of random numbers"
    if shape == []:
        shape = None
    return mt.random_sample(shape)

def uniform(minimum, maximum, shape=[]):
    """uniform(minimum, maximum, shape=[]) returns array of given shape of random reals
    in given range"""
    if shape == []:
        shape = None
    return mt.uniform(minimum, maximum, shape)

def randint(minimum, maximum=None, shape=[]):
    """randint(min, max, shape=[]) = random integers >=min, < max
    If max not given, random integers >= 0, <min"""
    if not isinstance(minimum, int):
        raise ArgumentError, "randint requires first argument integer"
    if maximum is None:
        maximum = minimum
        minimum = 0
    if not isinstance(maximum, int):
        raise ArgumentError, "randint requires second argument integer"
    a = ((maximum-minimum)* random(shape))
    if isinstance(a, Numeric.ndarray):
        return minimum + a.astype(Numeric.int)
    else:
        return minimum + int(a)

def random_integers(maximum, minimum=1, shape=[]):
    """random_integers(max, min=1, shape=[]) = random integers in range min-max inclusive"""
    return randint(minimum, maximum+1, shape)

def permutation(n):
    "permutation(n) = a permutation of indices range(n)"
    return mt.permutation(n)

def standard_normal(shape=[]):
    """standard_normal(n) or standard_normal([n, m, ...]) returns array of
           random numbers normally distributed with mean 0 and standard
           deviation 1"""
    if shape == []:
        shape = None
    return mt.standard_normal(shape)

def normal(mean, std, shape=[]):
    """normal(mean, std, n) or normal(mean, std, [n, m, ...]) returns
    array of random numbers randomly distributed with specified mean and
    standard deviation"""
    if shape == []:
        shape = None
    return mt.normal(mean, std, shape)

def multivariate_normal(mean, cov, shape=[]):
    """multivariate_normal(mean, cov) or multivariate_normal(mean, cov, [m, n, ...])
    returns an array containing multivariate normally distributed random numbers
    with specified mean and covariance.

    mean must be a 1 dimensional array. cov must be a square two dimensional
    array with the same number of rows and columns as mean has elements.

    The first form returns a single 1-D array containing a multivariate
    normal.

    The second form returns an array of shape (m, n, ..., cov.shape[0]).
    In this case, output[i,j,...,:] is a 1-D array containing a multivariate
    normal."""
    if shape == []:
        shape = None
    return mt.multivariate_normal(mean, cov, shape)

def exponential(mean, shape=[]):
    """exponential(mean, n) or exponential(mean, [n, m, ...]) returns array
      of random numbers exponentially distributed with specified mean"""
    if shape == []:
        shape = None
    return mt.exponential(mean, shape)

def beta(a, b, shape=[]):
    """beta(a, b) or beta(a, b, [n, m, ...]) returns array of beta distributed random numbers."""
    if shape == []:
        shape = None
    return mt.beta(a, b, shape)

def gamma(a, r, shape=[]):
    """gamma(a, r) or gamma(a, r, [n, m, ...]) returns array of gamma distributed random numbers."""
    if shape == []:
        shape = None
    return mt.gamma(a, r, shape)

def F(dfn, dfd, shape=[]):
    """F(dfn, dfd) or F(dfn, dfd, [n, m, ...]) returns array of F distributed random numbers with dfn degrees of freedom in the numerator and dfd degrees of freedom in the denominator."""
    if shape == []:
        shape = None
    return mt.f(dfn, dfd, shape)

def noncentral_F(dfn, dfd, nconc, shape=[]):
    """noncentral_F(dfn, dfd, nonc) or noncentral_F(dfn, dfd, nonc, [n, m, ...]) returns array of noncentral F distributed random numbers with dfn degrees of freedom in the numerator and dfd degrees of freedom in the denominator, and noncentrality parameter nconc."""
    if shape == []:
        shape = None
    return mt.noncentral_f(dfn, dfd, nconc, shape)

def chi_square(df, shape=[]):
    """chi_square(df) or chi_square(df, [n, m, ...]) returns array of chi squared distributed random numbers with df degrees of freedom."""
    if shape == []:
        shape = None
    return mt.chisquare(df, shape)

def noncentral_chi_square(df, nconc, shape=[]):
    """noncentral_chi_square(df, nconc) or chi_square(df, nconc, [n, m, ...]) returns array of noncentral chi squared distributed random numbers with df degrees of freedom and noncentrality parameter."""
    if shape == []:
        shape = None
    return mt.noncentral_chisquare(df, nconc, shape)

def binomial(trials, p, shape=[]):
    """binomial(trials, p) or binomial(trials, p, [n, m, ...]) returns array of binomially distributed random integers.

           trials is the number of trials in the binomial distribution.
           p is the probability of an event in each trial of the binomial distribution."""
    if shape == []:
        shape = None
    return mt.binomial(trials, p, shape)

def negative_binomial(trials, p, shape=[]):
    """negative_binomial(trials, p) or negative_binomial(trials, p, [n, m, ...]) returns
           array of negative binomially distributed random integers.

           trials is the number of trials in the negative binomial distribution.
           p is the probability of an event in each trial of the negative binomial distribution."""
    if shape == []:
        shape = None
    return mt.negative_binomial(trials, p, shape)

def multinomial(trials, probs, shape=[]):
    """multinomial(trials, probs) or multinomial(trials, probs, [n, m, ...]) returns
           array of multinomial distributed integer vectors.

           trials is the number of trials in each multinomial distribution.
           probs is a one dimensional array. There are len(prob)+1 events.
           prob[i] is the probability of the i-th event, 0<=i<len(prob).
           The probability of event len(prob) is 1.-Numeric.sum(prob).

       The first form returns a single 1-D array containing one multinomially
           distributed vector.

           The second form returns an array of shape (m, n, ..., len(probs)).
           In this case, output[i,j,...,:] is a 1-D array containing a multinomially
           distributed integer 1-D array."""
    if shape == []:
        shape = None
    return mt.multinomial(trials, probs, shape)

def poisson(mean, shape=[]):
    """poisson(mean) or poisson(mean, [n, m, ...]) returns array of poisson
           distributed random integers with specified mean."""
    if shape == []:
        shape = None
    return mt.poisson(mean, shape)

def mean_var_test(x, type, mean, var, skew=[]):
    n = len(x) * 1.0
    x_mean = Numeric.sum(x,axis=0)/n
    x_minus_mean = x - x_mean
    x_var = Numeric.sum(x_minus_mean*x_minus_mean,axis=0)/(n-1.0)
    print "\nAverage of ", len(x), type
    print "(should be about ", mean, "):", x_mean
    print "Variance of those random numbers (should be about ", var, "):", x_var
    if skew != []:
        x_skew = (Numeric.sum(x_minus_mean*x_minus_mean*x_minus_mean,axis=0)/9998.)/x_var**(3./2.)
        print "Skewness of those random numbers (should be about ", skew, "):", x_skew

def test():
    obj = mt.get_state()
    mt.set_state(obj)
    obj2 = mt.get_state()
    if (obj2[1] - obj[1]).any():
        raise SystemExit, "Failed seed test."
    print "First random number is", random()
    print "Average of 10000 random numbers is", Numeric.sum(random(10000),axis=0)/10000.
    x = random([10,1000])
    if len(x.shape) != 2 or x.shape[0] != 10 or x.shape[1] != 1000:
        raise SystemExit, "random returned wrong shape"
    x.shape = (10000,)
    print "Average of 100 by 100 random numbers is", Numeric.sum(x,axis=0)/10000.
    y = uniform(0.5,0.6, (1000,10))
    if len(y.shape) !=2 or y.shape[0] != 1000 or y.shape[1] != 10:
        raise SystemExit, "uniform returned wrong shape"
    y.shape = (10000,)
    if Numeric.minimum.reduce(y) <= 0.5 or Numeric.maximum.reduce(y) >= 0.6:
        raise SystemExit, "uniform returned out of desired range"
    print "randint(1, 10, shape=[50])"
    print randint(1, 10, shape=[50])
    print "permutation(10)", permutation(10)
    print "randint(3,9)", randint(3,9)
    print "random_integers(10, shape=[20])"
    print random_integers(10, shape=[20])
    s = 3.0
    x = normal(2.0, s, [10, 1000])
    if len(x.shape) != 2 or x.shape[0] != 10 or x.shape[1] != 1000:
        raise SystemExit, "standard_normal returned wrong shape"
    x.shape = (10000,)
    mean_var_test(x, "normally distributed numbers with mean 2 and variance %f"%(s**2,), 2, s**2, 0)
    x = exponential(3, 10000)
    mean_var_test(x, "random numbers exponentially distributed with mean %f"%(s,), s, s**2, 2)
    x = multivariate_normal(Numeric.array([10,20]), Numeric.array(([1,2],[2,4])))
    print "\nA multivariate normal", x
    if x.shape != (2,): raise SystemExit, "multivariate_normal returned wrong shape"
    x = multivariate_normal(Numeric.array([10,20]), Numeric.array([[1,2],[2,4]]), [4,3])
    print "A 4x3x2 array containing multivariate normals"
    print x
    if x.shape != (4,3,2): raise SystemExit, "multivariate_normal returned wrong shape"
    x = multivariate_normal(Numeric.array([-100,0,100]), Numeric.array([[3,2,1],[2,2,1],[1,1,1]]), 10000)
    x_mean = Numeric.sum(x,axis=0)/10000.
    print "Average of 10000 multivariate normals with mean [-100,0,100]"
    print x_mean
    x_minus_mean = x - x_mean
    print "Estimated covariance of 10000 multivariate normals with covariance [[3,2,1],[2,2,1],[1,1,1]]"
    print Numeric.dot(Numeric.transpose(x_minus_mean),x_minus_mean)/9999.
    x = beta(5.0, 10.0, 10000)
    mean_var_test(x, "beta(5.,10.) random numbers", 0.333, 0.014)
    x = gamma(.01, 2., 10000)
    mean_var_test(x, "gamma(.01,2.) random numbers", 2*100, 2*100*100)
    x = chi_square(11., 10000)
    mean_var_test(x, "chi squared random numbers with 11 degrees of freedom", 11, 22, 2*Numeric.sqrt(2./11.))
    x = F(5., 10., 10000)
    mean_var_test(x, "F random numbers with 5 and 10 degrees of freedom", 1.25, 1.35)
    x = poisson(50., 10000)
    mean_var_test(x, "poisson random numbers with mean 50", 50, 50, 0.14)
    print "\nEach element is the result of 16 binomial trials with probability 0.5:"
    print binomial(16, 0.5, 16)
    print "\nEach element is the result of 16 negative binomial trials with probability 0.5:"
    print negative_binomial(16, 0.5, [16,])
    print "\nEach row is the result of 16 multinomial trials with probabilities [0.1, 0.5, 0.1 0.3]:"
    x = multinomial(16, [0.1, 0.5, 0.1], 8)
    print x
    print "Mean = ", Numeric.sum(x,axis=0)/8.

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>import sys
import numpy as np
import pandas as pd
from natsort import natsorted

from pyranges.statistics import StatisticsMethods
from pyranges.genomicfeatures import GenomicFeaturesMethods
from pyranges import PyRanges
from pyranges.helpers import single_value_key, get_key_from_df

def set_dtypes(df, int64):

    # if extended is None:
    #     extended = False if df.Start.dtype == np.int32 else True

    if not int64:
        dtypes = {
            "Start": np.int32,
            "End": np.int32,
            "Chromosome": "category",
            "Strand": "category"
        }
    else:
        dtypes = {
            "Start": np.int64,
            "End": np.int64,
            "Chromosome": "category",
            "Strand": "category"
        }

    if not "Strand" in df:
        del dtypes["Strand"]

    # need to ascertain that object columns do not consist of multiple types
    # https://github.com/biocore-ntnu/epic2/issues/32
    for column in "Chromosome Strand".split():
        if not column in df:
            continue

        if df[column].dtype == object and len(
                df[column].apply(type).drop_duplicates()) > 1:
            df[column] = df[column].astype(str)
        elif df[column].dtype != object:
            df[column] = df[column].astype(str)

    for col, dtype in dtypes.items():

        if df[col].dtype.name != dtype:

            df[col] = df[col].astype(dtype)

    return df

def create_df_dict(df, stranded):

    chrs = df.Chromosome.cat.remove_unused_categories()

    df["Chromosome"] = chrs

    if stranded:
        grpby_key = "Chromosome Strand".split()
        df["Strand"] = df.Strand.cat.remove_unused_categories()
    else:
        grpby_key = "Chromosome"

    return {k: v for k, v in df.groupby(grpby_key)}

def create_pyranges_df(chromosomes, starts, ends, strands=None):

    if isinstance(chromosomes, str) or isinstance(chromosomes, int):
        chromosomes = pd.Series([chromosomes] * len(starts), dtype="category")

    if strands is not None:

        if isinstance(strands, str):
            strands = pd.Series([strands] * len(starts), dtype="category")

        columns = [chromosomes, starts, ends, strands]
        lengths = list(str(len(s)) for s in columns)
        assert len(
            set(lengths)
        ) == 1, "chromosomes, starts, ends and strands must be of equal length. But are {}".format(
            ", ".join(lengths))
        colnames = "Chromosome Start End Strand".split()
    else:
        columns = [chromosomes, starts, ends]
        lengths = list(str(len(s)) for s in columns)
        assert len(
            set(lengths)
        ) == 1, "chromosomes, starts and ends must be of equal length. But are {}".format(
            ", ".join(lengths))
        colnames = "Chromosome Start End".split()

    idx = range(len(starts))
    series_to_concat = []
    for s in columns:
        if isinstance(s, pd.Series):
            s = pd.Series(s.values, index=idx)
        else:
            s = pd.Series(s, index=idx)

        series_to_concat.append(s)

    df = pd.concat(series_to_concat, axis=1)
    df.columns = colnames

    return df

def check_strandedness(df):
    """Check whether strand contains '.'"""

    if "Strand" not in df:
        return False

    contains_more_than_plus_minus_in_strand_col = False

    if str(df.Strand.dtype) == "category" and (
            set(df.Strand.cat.categories) - set("+-")):
        contains_more_than_plus_minus_in_strand_col = True
    elif not ((df.Strand == "+") | (df.Strand == "-")).all():
        contains_more_than_plus_minus_in_strand_col = True

        # if contains_more_than_plus_minus_in_strand_col:
        #     logging.warning("Strand contained more symbols than '+' or '-'. Not supported (yet) in PyRanges.")

    return not contains_more_than_plus_minus_in_strand_col

def _init(self,
          df=None,
          chromosomes=None,
          starts=None,
          ends=None,
          strands=None,
          int64=False,
          copy_df=True):
    # TODO: add categorize argument with dict of args to categorize?

    if isinstance(df, PyRanges):
        raise Exception("Object is already a PyRange.")

    if isinstance(df, pd.DataFrame):
        assert all(
            c in df for c in "Chromosome Start End".split()
        ), "The dataframe does not have all the columns Chromosome, Start and End."
        if copy_df:
            df = df.copy()

    if df is False or df is None:
        df = create_pyranges_df(chromosomes, starts, ends, strands)

    if isinstance(df, pd.DataFrame):

        df = df.reset_index(drop=True)

        stranded = check_strandedness(df)

        df = set_dtypes(df, int64)

        self.__dict__["dfs"] = create_df_dict(df, stranded)

    # df is actually dict of dfs
    else:

        empty_removed = {k: v.copy() for k, v in df.items() if not v.empty}

        _single_value_key = True
        _key_same = True
        _strand_valid = True
        _has_strand = True
        for key, df in empty_removed.items():

            _key = get_key_from_df(df)
            _single_value_key = single_value_key(df) and _single_value_key
            _key_same = (_key == key) and _key_same

            if isinstance(_key, tuple):
                _strand_valid = _strand_valid and (_key[1] in ["+", "-"])
            else:
                _has_strand = False

        if not all([_single_value_key, _key_same, _strand_valid]):
            df = pd.concat(empty_removed.values()).reset_index(drop=True)

            if _has_strand and _strand_valid:
                empty_removed = df.groupby(["Chromosome", "Strand"])
            else:
                empty_removed = df.groupby("Chromosome")

            empty_removed = {k: v for (k, v) in empty_removed}

        self.__dict__["dfs"] = empty_removed

    self.__dict__["features"] = GenomicFeaturesMethods(self)
    self.__dict__["stats"] = StatisticsMethods(self)
<commit_msg>#159 Chromosome and Strand should be str (#162)<commit_after>import sys
import numpy as np
import pandas as pd
from natsort import natsorted

from pyranges.statistics import StatisticsMethods
from pyranges.genomicfeatures import GenomicFeaturesMethods
from pyranges import PyRanges
from pyranges.helpers import single_value_key, get_key_from_df

def set_dtypes(df, int64):

    # if extended is None:
    #     extended = False if df.Start.dtype == np.int32 else True

    if not int64:
        dtypes = {
            "Start": np.int32,
            "End": np.int32,
            "Chromosome": "category",
            "Strand": "category",
        }
    else:
        dtypes = {
            "Start": np.int64,
            "End": np.int64,
            "Chromosome": "category",
            "Strand": "category",
        }

    if "Strand" not in df:
        del dtypes["Strand"]

    # need to ascertain that object columns do not consist of multiple types
    # https://github.com/biocore-ntnu/epic2/issues/32
    for column in "Chromosome Strand".split():
        if column not in df:
            continue
        df[column] = df[column].astype(str)

    for col, dtype in dtypes.items():
        if df[col].dtype.name != dtype:
            df[col] = df[col].astype(dtype)
    return df

def create_df_dict(df, stranded):

    chrs = df.Chromosome.cat.remove_unused_categories()

    df["Chromosome"] = chrs

    if stranded:
        grpby_key = "Chromosome Strand".split()
        df["Strand"] = df.Strand.cat.remove_unused_categories()
    else:
        grpby_key = "Chromosome"

    return {k: v for k, v in df.groupby(grpby_key)}

def create_pyranges_df(chromosomes, starts, ends, strands=None):

    if isinstance(chromosomes, str) or isinstance(chromosomes, int):
        chromosomes = pd.Series([chromosomes] * len(starts), dtype="category")

    if strands is not None:

        if isinstance(strands, str):
            strands = pd.Series([strands] * len(starts), dtype="category")

        columns = [chromosomes, starts, ends, strands]
        lengths = list(str(len(s)) for s in columns)
        assert (
            len(set(lengths)) == 1
        ), "chromosomes, starts, ends and strands must be of equal length. But are {}".format(
            ", ".join(lengths)
        )
        colnames = "Chromosome Start End Strand".split()
    else:
        columns = [chromosomes, starts, ends]
        lengths = list(str(len(s)) for s in columns)
        assert (
            len(set(lengths)) == 1
        ), "chromosomes, starts and ends must be of equal length. But are {}".format(
            ", ".join(lengths)
        )
        colnames = "Chromosome Start End".split()

    idx = range(len(starts))
    series_to_concat = []
    for s in columns:
        if isinstance(s, pd.Series):
            s = pd.Series(s.values, index=idx)
        else:
            s = pd.Series(s, index=idx)

        series_to_concat.append(s)

    df = pd.concat(series_to_concat, axis=1)
    df.columns = colnames

    return df

def check_strandedness(df):
    """Check whether strand contains '.'"""

    if "Strand" not in df:
        return False

    contains_more_than_plus_minus_in_strand_col = False

    if str(df.Strand.dtype) == "category" and (
        set(df.Strand.cat.categories) - set("+-")
    ):
        contains_more_than_plus_minus_in_strand_col = True
    elif not ((df.Strand == "+") | (df.Strand == "-")).all():
        contains_more_than_plus_minus_in_strand_col = True

        # if contains_more_than_plus_minus_in_strand_col:
        #     logging.warning("Strand contained more symbols than '+' or '-'. Not supported (yet) in PyRanges.")

    return not contains_more_than_plus_minus_in_strand_col

def _init(
    self,
    df=None,
    chromosomes=None,
    starts=None,
    ends=None,
    strands=None,
    int64=False,
    copy_df=True,
):
    # TODO: add categorize argument with dict of args to categorize?

    if isinstance(df, PyRanges):
        raise Exception("Object is already a PyRange.")

    if isinstance(df, pd.DataFrame):
        assert all(
            c in df for c in "Chromosome Start End".split()
        ), "The dataframe does not have all the columns Chromosome, Start and End."
        if copy_df:
            df = df.copy()

    if df is False or df is None:
        df = create_pyranges_df(chromosomes, starts, ends, strands)

    if isinstance(df, pd.DataFrame):

        df = df.reset_index(drop=True)

        stranded = check_strandedness(df)

        df = set_dtypes(df, int64)

        self.__dict__["dfs"] = create_df_dict(df, stranded)

    # df is actually dict of dfs
    else:

        empty_removed = {k: v.copy() for k, v in df.items() if not v.empty}

        _single_value_key = True
        _key_same = True
        _strand_valid = True
        _has_strand = True
        for key, df in empty_removed.items():

            _key = get_key_from_df(df)
            _single_value_key = single_value_key(df) and _single_value_key
            _key_same = (_key == key) and _key_same

            if isinstance(_key, tuple):
                _strand_valid = _strand_valid and (_key[1] in ["+", "-"])
            else:
                _has_strand = False

        if not all([_single_value_key, _key_same, _strand_valid]):
            df = pd.concat(empty_removed.values()).reset_index(drop=True)

            if _has_strand and _strand_valid:
                empty_removed = df.groupby(["Chromosome", "Strand"])
            else:
                empty_removed = df.groupby("Chromosome")

            empty_removed = {k: v for (k, v) in empty_removed}

        self.__dict__["dfs"] = empty_removed

    self.__dict__["features"] = GenomicFeaturesMethods(self)
    self.__dict__["stats"] = StatisticsMethods(self)
<|endoftext|>