<commit_before>import numpy as nm
import libxml2
from array import *
from libxml2 import xmlAttr
import matplotlib.pyplot as plt

eVdoc = libxml2.parseFile("./eV.xml")
ctxt = eVdoc.xpathNewContext()

Volume=nm.array(map(float,map(xmlAttr.getContent,ctxt.xpathEval("//@volume"))))
totalEnergy=nm.array(map(float,map(xmlAttr.getContent,ctxt.xpathEval("//@totalEnergy"))))

p=nm.polyfit(Volume,totalEnergy,2)
curve=nm.poly1d(p)
xa = nm.linspace(Volume[0],Volume[-1],100)
print xa
print curve(xa)
plt.figure(1)
plt.title('Aluminum Volume')
plt.ylabel(r'total energy in $[Hartree]$')
plt.xlabel(r'volume in $[Bohr]^3$')
plt.plot(xa,curve(xa),'o')
#plt.plot(Volume,totalEnergy,'o')

plt.show()<commit_msg>Ag Matplotlib script<commit_after>import numpy as nm
import libxml2
from array import *
from libxml2 import xmlAttr
import matplotlib.pyplot as plt

eVdoc = libxml2.parseFile("./eV.xml")
ctxt = eVdoc.xpathNewContext()

Volume=nm.array(map(float,map(xmlAttr.getContent,ctxt.xpathEval("//@volume"))))
totalEnergy=nm.array(map(float,map(xmlAttr.getContent,ctxt.xpathEval("//@totalEnergy"))))

p=nm.polyfit(Volume,totalEnergy,2)
curve=nm.poly1d(p)
min=nm.roots(nm.polyder(p))

xa = nm.linspace(Volume[0],Volume[-1],100)

plt.figure(1)
plt.title('Aluminum Volume')
plt.ylabel(r'total energy in $[Hartree]$')
plt.xlabel(r'volume in $[Bohr]^3$')
plt.plot(xa,curve(xa),'-')
plt.plot(Volume,totalEnergy,'o')
plt.plot(min,curve(min),'o')
plt.annotate('minimumn Volume '+str(min[0]), xy=(min,curve(min)), xycoords='data' , xytext=(min-10,curve(min)+0.002) ,  arrowprops=dict(arrowstyle="->"))
print 'minimumn Volume '+str(min[0])
plt.show()<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: expandtab:tabstop=4:shiftwidth=4
'''
  Process OpenShift event stream
'''
#
#   Copyright 2016 Red Hat Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#This is not a module, but pylint thinks it is.  This is a command.
#pylint: disable=invalid-name
# pylint flaggs import errors, as the bot doesn't know out openshift-tools libs
#pylint: disable=import-error

import argparse
from openshift_tools.monitoring.metric_sender import MetricSender
import json
from Queue import Queue
import re
import subprocess
import threading
import time
import yaml

#pylint: disable=too-few-public-methods
class OpenshiftEventConsumer(object):
    ''' Submits events to Zabbix '''

    def __init__(self, args, queue, zbx_keys):
        self.queue = queue
        self.args = args
        self.zbx_keys = zbx_keys

    def run(self):
        ''' main function '''

        while True:
            event_list = []
            while not self.queue.empty():
                event = self.queue.get()
                if self.args.debug:
                    print "Processing event: {}".format(str(event))
                event_list.append(event)

            # initialize event counts so that we send '0' events
            # in the case where no events were received
            event_counts = {}
            for zbx_key in self.zbx_keys:
                event_counts[zbx_key] = 0

            # add up each distinct event
            for event in event_list:
                event_counts[event] += 1

            if self.args.verbose or self.args.dry_run:
                print "Got events: " + str(event_counts)

            if not self.args.dry_run:
                metric_sender = MetricSender(verbose=self.args.verbose, debug=self.args.debug)
                for event, count in event_counts.iteritems():
                    metric_sender.add_metric({event: count})
                metric_sender.send_metrics()

            time.sleep(self.args.reporting_period)

        # Never should get here

class OpenshiftEventWatcher(object):
    ''' Watches OpenShift event stream '''

    def __init__(self, queue):
        self.args = None
        self.queue = queue
        self.parse_args()

    def run(self):
        '''  Main function '''

        self.event_watch_loop()

    def watch_list_setup(self):
        ''' create dict of events/reasons to watch for
            plus a regex to further filter events'''
        with open(self.args.config, 'r') as config:
            self.args.watch_for = yaml.load(config)['event_watcher_config']

    def parse_args(self):
        ''' parse the args from the cli '''

        parser = argparse.ArgumentParser(description='OpenShift event watcher')
        parser.add_argument('--kubeconfig', default='/etc/origin/master/admin.kubeconfig',
                            help='Location of OpenShift kubeconfig file')
        parser.add_argument('--config', default='/container_setup/monitoring-config.yml',
                            help='Config file for event watcher script')
        parser.add_argument('-v', '--verbose', action='store_true',
                            default=None, help='Verbose?')
        parser.add_argument('--debug', action='store_true',
                            default=None, help='Debug?')
        parser.add_argument('--dry-run', action='store_true', default=False,
                            help='Do not send results to Zabbix')
        parser.add_argument('--reporting-period', default=60, type=int,
                            help='How many seconds between each reporting period')

        self.args = parser.parse_args()

        self.watch_list_setup()

    def check_event(self, event):
        ''' If an event is something we're looking for
            return the key it should be reported as '''

        # Most events aren't something we will care about
        # so catch that case and return early
        foundReason = ''
        for reasonCandidate in event['reasonFields']:
            if reasonCandidate in self.args.watch_for.keys():
                foundReason = reasonCandidate
        if not foundReason:
            return None

        regex_list = self.args.watch_for[foundReason]
        for regex in regex_list:
            if re.search(regex['pattern'], event['message']):
                return regex['zbx_key']

        # If we made it here, then there was no regex match
        # so the event is not something we will report to zabbix
        return None

    def get_zbx_keys(self):
        ''' return list of zbx keys config file says to report on '''

        zbx_keys = []
        for _, regex_list in self.args.watch_for.iteritems():
            for regex in regex_list:
                zbx_keys.append(regex['zbx_key'])

        return zbx_keys

    def event_watch_loop(self):
        ''' Loop to read/process OpenShift events '''

        while True:
            # k8s v1.11 has a bug preventing output of watches in non-default
            # format types, so we can't use -o here.
            popen = subprocess.Popen(['oc', 'get', 'events', '--all-namespaces',
                                      '--config', self.args.kubeconfig,
                                      '--watch-only'], bufsize=1,
                                      stdout=subprocess.PIPE)

            json_str = ""
            print "Watching for events: " + str(self.args.watch_for)

            for line in iter(popen.stdout.readline, b''):
                # The 'Reason' field should be the fourth element in
                # the line, and the previous three elements do not contain
                # values that allow whitespace - a split will suffice.

                fields = line.split()
                # We need to do a little checking to know where to
                # find the 'message' column
                if len(fields) > 8:
                    event = {
                        'reasonFields': fields[7:9],
                        'message': ' '.join(fields[8:])
                    }

                    if self.args.debug:
                        print "Event type candidates: " + ' '.join(event['reasonFields'])
                        print line

                    result = self.check_event(event)
                    if result:
                        if self.args.verbose:
                            print "Matched event: " + ' '.join(event['reasonFields']) + \
                                  " " + event['message']
                        self.queue.put(result)

            # Never should get here - but if it does, add a cool-off timer for a minute
            # to avoid smashing repeated 'oc get events' calls.
            time.sleep(self.args.reporting_period)

if __name__ == '__main__':
    event_queue = Queue()

    OEW = OpenshiftEventWatcher(event_queue)
    zbx_key_list = OEW.get_zbx_keys()
    watch_thread = threading.Thread(target=OEW.run)
    watch_thread.start()

    OEC = OpenshiftEventConsumer(OEW.args, event_queue, zbx_key_list)
    event_consumer = threading.Thread(target=OEC.run)
    event_consumer.start()
<commit_msg>pylint corrections for cron-event-watcher OSD-3225<commit_after>#!/usr/bin/env python
# vim: expandtab:tabstop=4:shiftwidth=4
'''
  Process OpenShift event stream
'''
#
#   Copyright 2016 Red Hat Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#This is not a module, but pylint thinks it is.  This is a command.
#pylint: disable=invalid-name
# pylint flaggs import errors, as the bot doesn't know out openshift-tools libs
#pylint: disable=import-error

import argparse
from Queue import Queue
import re
import subprocess
import threading
import time
import yaml
from openshift_tools.monitoring.metric_sender import MetricSender

#pylint: disable=too-few-public-methods
class OpenshiftEventConsumer(object):
    ''' Submits events to Zabbix '''

    def __init__(self, args, queue, zbx_keys):
        self.queue = queue
        self.args = args
        self.zbx_keys = zbx_keys

    def run(self):
        ''' main function '''

        while True:
            event_list = []
            while not self.queue.empty():
                event = self.queue.get()
                if self.args.debug:
                    print "Processing event: {}".format(str(event))
                event_list.append(event)

            # initialize event counts so that we send '0' events
            # in the case where no events were received
            event_counts = {}
            for zbx_key in self.zbx_keys:
                event_counts[zbx_key] = 0

            # add up each distinct event
            for event in event_list:
                event_counts[event] += 1

            if self.args.verbose or self.args.dry_run:
                print "Got events: " + str(event_counts)

            if not self.args.dry_run:
                metric_sender = MetricSender(verbose=self.args.verbose, debug=self.args.debug)
                for event, count in event_counts.iteritems():
                    metric_sender.add_metric({event: count})
                metric_sender.send_metrics()

            time.sleep(self.args.reporting_period)

        # Never should get here

class OpenshiftEventWatcher(object):
    ''' Watches OpenShift event stream '''

    def __init__(self, queue):
        self.args = None
        self.queue = queue
        self.parse_args()

    def run(self):
        '''  Main function '''

        self.event_watch_loop()

    def watch_list_setup(self):
        ''' create dict of events/reasons to watch for
            plus a regex to further filter events'''
        with open(self.args.config, 'r') as config:
            self.args.watch_for = yaml.load(config)['event_watcher_config']

    def parse_args(self):
        ''' parse the args from the cli '''

        parser = argparse.ArgumentParser(description='OpenShift event watcher')
        parser.add_argument('--kubeconfig', default='/etc/origin/master/admin.kubeconfig',
                            help='Location of OpenShift kubeconfig file')
        parser.add_argument('--config', default='/container_setup/monitoring-config.yml',
                            help='Config file for event watcher script')
        parser.add_argument('-v', '--verbose', action='store_true',
                            default=None, help='Verbose?')
        parser.add_argument('--debug', action='store_true',
                            default=None, help='Debug?')
        parser.add_argument('--dry-run', action='store_true', default=False,
                            help='Do not send results to Zabbix')
        parser.add_argument('--reporting-period', default=60, type=int,
                            help='How many seconds between each reporting period')

        self.args = parser.parse_args()

        self.watch_list_setup()

    def check_event(self, event):
        ''' If an event is something we're looking for
            return the key it should be reported as '''

        # Most events aren't something we will care about
        # so catch that case and return early
        foundReason = ''
        for reasonCandidate in event['reasonFields']:
            if reasonCandidate in self.args.watch_for.keys():
                foundReason = reasonCandidate
        if not foundReason:
            return None

        regex_list = self.args.watch_for[foundReason]
        for regex in regex_list:
            if re.search(regex['pattern'], event['message']):
                return regex['zbx_key']

        # If we made it here, then there was no regex match
        # so the event is not something we will report to zabbix
        return None

    def get_zbx_keys(self):
        ''' return list of zbx keys config file says to report on '''

        zbx_keys = []
        for _, regex_list in self.args.watch_for.iteritems():
            for regex in regex_list:
                zbx_keys.append(regex['zbx_key'])

        return zbx_keys

    def event_watch_loop(self):
        ''' Loop to read/process OpenShift events '''

        while True:
            # k8s v1.11 has a bug preventing output of watches in non-default
            # format types, so we can't use -o here.
            popen = subprocess.Popen(['oc', 'get', 'events', '--all-namespaces',
                                      '--config', self.args.kubeconfig,
                                      '--watch-only'], bufsize=1,
                                     stdout=subprocess.PIPE)

            print "Watching for events: " + str(self.args.watch_for)

            for line in iter(popen.stdout.readline, b''):
                # The 'Reason' field should be the fourth element in
                # the line, and the previous three elements do not contain
                # values that allow whitespace - a split will suffice.

                fields = line.split()
                # We need to do a little checking to know where to
                # find the 'message' column
                if len(fields) > 8:
                    event = {
                        'reasonFields': fields[7:9],
                        'message': ' '.join(fields[8:])
                    }

                    if self.args.debug:
                        print "Event type candidates: " + ' '.join(event['reasonFields'])
                        print line

                    result = self.check_event(event)
                    if result:
                        if self.args.verbose:
                            print "Matched event: " + ' '.join(event['reasonFields']) + \
                                  " " + event['message']
                        self.queue.put(result)

            # Never should get here - but if it does, add a cool-off timer for a minute
            # to avoid smashing repeated 'oc get events' calls.
            time.sleep(self.args.reporting_period)

if __name__ == '__main__':
    event_queue = Queue()

    OEW = OpenshiftEventWatcher(event_queue)
    zbx_key_list = OEW.get_zbx_keys()
    watch_thread = threading.Thread(target=OEW.run)
    watch_thread.start()

    OEC = OpenshiftEventConsumer(OEW.args, event_queue, zbx_key_list)
    event_consumer = threading.Thread(target=OEC.run)
    event_consumer.start()
<|endoftext|>


<commit_before>import math
from collections import deque

class Dgim(object):
    """An implementation of the DGIM algorithm.
    It estimates the number of 1s present in a sliding window of a stream
    while using few memory.

    The algorithm is described in:
    Datar, Mayur, et al. "Maintaining stream statistics over sliding windows."
    SIAM Journal on Computing 31.6 (2002): 1794-1813.

    An explanation of the algorithm can also be found here:
    http://infolab.stanford.edu/~ullman/mmds/ch4.pdf
    """

    def __init__(self, N, r=2):
        """Constructor
        :param N: size of the sliding window
        :type N: int
        :param r: the maximum number of buckets of the same size
        :type r: int
        """
        self.N = N
        if r < 2:
            raise ValueError("'r' should be higher or equal to 2. Got {}.".format(r))
        self.r = r
        self.queues = []
        if N == 0:
            max_index = -1
        else:
            max_index = int(math.ceil(math.log(N)/math.log(2)))

        for i in range(max_index + 1):
            self.queues.append(deque())

        self.timestamp = 0

    @property
    def error_rate(self):
        """Return the maximum error rate made by the algorithm.
        Let c be the true result and e the estimate returned by the dgim
        algorithm.
        abs(c-e) < error_rate * c
        :returns: float
        """
        return 1/float(self.r)

    @property
    def nb_buckets(self):
        """Returns the number of buckets.
        :returns: int
        """
        result = 0
        for queue in self.queues:
            result += len(queue)
        return result

    def update(self, elt):
        """Update the stream with one element.
        The element can be either 0 or 1.
        :param elt: the latest element of the stream
        :type elt: int
        """
        self.timestamp += 1
        #check if oldest bucket should be removed
        if self.is_oldest_bucket_too_old():
            self.drop_oldest_bucket()
        if elt != 1:
            #nothing to do
            return

        reminder = self.timestamp
        for queue in self.queues:
            queue.appendleft(reminder)
            if len(queue) <= self.r:
                break
            last = queue.pop()
            second_last = queue.pop()
            # merge last two buckets.
            reminder = max(last, second_last)

    def is_oldest_bucket_too_old(self):
        """Check if the latest bucket is too old and should be dropped.
        :returns: bool
        """
        oldest_bucket_timestamp = self.get_oldest_bucket_timestamp()
        return (oldest_bucket_timestamp >= 0 and
                oldest_bucket_timestamp <= self.timestamp - self.N)

    def drop_oldest_bucket(self):
        """Drop oldest bucket timestamp."""
        for queue in reversed(self.queues):
            if len(queue) > 0:
                queue.pop()
                break

    def get_oldest_bucket_timestamp(self):
        """Return the timestamp of the oldest bucket.
        If there is no bucket, returns -1
        :returns: int
        """
        for queue in reversed(self.queues):
            if len(queue) > 0:
                return queue[-1]
        return -1

    def get_count(self):
        """Returns an estimate of the number of ones in the sliding window.
        :returns: int
        """
        result = 0
        max_value = 0
        power_of_two = 1
        for queue in self.queues:
            queue_length = len(queue)
            if queue_length > 0:
                max_value = power_of_two
                result += queue_length * power_of_two
            power_of_two = power_of_two << 1
        result -= math.floor(max_value/2)
        return int(result)
<commit_msg>Replace is_oldest_bucket_too_old() with is_oldest_bucket_too_old()<commit_after>import math
from collections import deque

class Dgim(object):
    """An implementation of the DGIM algorithm.
    It estimates the number of 1s present in a sliding window of a stream
    while using few memory.

    The algorithm is described in:
    Datar, Mayur, et al. "Maintaining stream statistics over sliding windows."
    SIAM Journal on Computing 31.6 (2002): 1794-1813.

    An explanation of the algorithm can also be found here:
    http://infolab.stanford.edu/~ullman/mmds/ch4.pdf
    """

    def __init__(self, N, r=2):
        """Constructor
        :param N: size of the sliding window
        :type N: int
        :param r: the maximum number of buckets of the same size
        :type r: int
        """
        self.N = N
        if r < 2:
            raise ValueError("'r' should be higher or equal to 2. Got {}.".format(r))
        self.r = r
        self.queues = []
        if N == 0:
            max_index = -1
        else:
            max_index = int(math.ceil(math.log(N)/math.log(2)))

        for i in range(max_index + 1):
            self.queues.append(deque())

        self.timestamp = 0

    @property
    def error_rate(self):
        """Return the maximum error rate made by the algorithm.
        Let c be the true result and e the estimate returned by the dgim
        algorithm.
        abs(c-e) < error_rate * c
        :returns: float
        """
        return 1/float(self.r)

    @property
    def nb_buckets(self):
        """Returns the number of buckets.
        :returns: int
        """
        result = 0
        for queue in self.queues:
            result += len(queue)
        return result

    def update(self, elt):
        """Update the stream with one element.
        The element can be either 0 or 1.
        :param elt: the latest element of the stream
        :type elt: int
        """
        self.timestamp += 1
        #check if oldest bucket should be removed
        if self.is_bucket_too_old(self.get_oldest_bucket_timestamp()):
            self.drop_oldest_bucket()
        if elt != 1:
            #nothing to do
            return

        reminder = self.timestamp
        for queue in self.queues:
            queue.appendleft(reminder)
            if len(queue) <= self.r:
                break
            last = queue.pop()
            second_last = queue.pop()
            # merge last two buckets.
            reminder = max(last, second_last)

    def is_bucket_too_old(self, bucket_timestamp):
        """Check if a bucket is too old and should be dropped.
        ;param bucket_timestamp: the bucket timestamp
        :type bucket_timestamp: int
        :returns: bool
        """
        return (bucket_timestamp >= 0 and
                bucket_timestamp <= self.timestamp - self.N)

    def drop_oldest_bucket(self):
        """Drop oldest bucket timestamp."""
        for queue in reversed(self.queues):
            if len(queue) > 0:
                queue.pop()
                break

    def get_oldest_bucket_timestamp(self):
        """Return the timestamp of the oldest bucket.
        If there is no bucket, returns -1
        :returns: int
        """
        for queue in reversed(self.queues):
            if len(queue) > 0:
                return queue[-1]
        return -1

    def get_count(self):
        """Returns an estimate of the number of ones in the sliding window.
        :returns: int
        """
        result = 0
        max_value = 0
        power_of_two = 1
        for queue in self.queues:
            queue_length = len(queue)
            if queue_length > 0:
                max_value = power_of_two
                result += queue_length * power_of_two
            power_of_two = power_of_two << 1
        result -= math.floor(max_value/2)
        return int(result)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright 2020 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
import os
import sys

from ros_buildfarm.argument import \
    add_argument_distribution_repository_key_files
from ros_buildfarm.argument import add_argument_distribution_repository_urls
from ros_buildfarm.argument import add_argument_dockerfile_dir
from ros_buildfarm.argument import add_argument_os_code_name
from ros_buildfarm.argument import add_argument_os_name
from ros_buildfarm.argument import add_argument_package_name
from ros_buildfarm.argument import add_argument_rosdistro_index_url
from ros_buildfarm.argument import add_argument_rosdistro_name
from ros_buildfarm.argument import add_argument_sourcepkg_dir
from ros_buildfarm.common import get_distribution_repository_keys
from ros_buildfarm.common import get_user_id
from ros_buildfarm.templates import create_dockerfile
from ros_buildfarm.templates import expand_template

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Run the 'sourcerpm' job")
    add_argument_rosdistro_index_url(parser, required=True)
    add_argument_rosdistro_name(parser)
    add_argument_package_name(parser)
    add_argument_os_name(parser)
    add_argument_os_code_name(parser)
    add_argument_dockerfile_dir(parser)
    add_argument_distribution_repository_urls(parser)
    add_argument_distribution_repository_key_files(parser)
    add_argument_sourcepkg_dir(parser)
    args = parser.parse_args(argv)

    data = copy.deepcopy(args.__dict__)
    data.update({
        'distribution_repository_urls': args.distribution_repository_urls,
        'distribution_repository_keys': get_distribution_repository_keys(
            args.distribution_repository_urls,
            args.distribution_repository_key_files),
        'target_repository': None,

        'uid': get_user_id(),
    })
    create_dockerfile(
        'release/rpm/sourcepkg_task.Dockerfile.em', data, args.dockerfile_dir)

    with open(os.path.join(args.dockerfile_dir, 'mock_config.cfg'), 'w') as mock_cfg:
        mock_cfg.write(expand_template('release/rpm/mock_config.cfg.em', data))

if __name__ == '__main__':
    main()
<commit_msg>Fix source RPM job regression (#759)<commit_after>#!/usr/bin/env python3

# Copyright 2020 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
import os
import sys

from ros_buildfarm.argument import \
    add_argument_distribution_repository_key_files
from ros_buildfarm.argument import add_argument_distribution_repository_urls
from ros_buildfarm.argument import add_argument_dockerfile_dir
from ros_buildfarm.argument import add_argument_os_code_name
from ros_buildfarm.argument import add_argument_os_name
from ros_buildfarm.argument import add_argument_package_name
from ros_buildfarm.argument import add_argument_rosdistro_index_url
from ros_buildfarm.argument import add_argument_rosdistro_name
from ros_buildfarm.argument import add_argument_sourcepkg_dir
from ros_buildfarm.common import get_distribution_repository_keys
from ros_buildfarm.common import get_user_id
from ros_buildfarm.templates import create_dockerfile
from ros_buildfarm.templates import expand_template

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Run the 'sourcerpm' job")
    add_argument_rosdistro_index_url(parser, required=True)
    add_argument_rosdistro_name(parser)
    add_argument_package_name(parser)
    add_argument_os_name(parser)
    add_argument_os_code_name(parser)
    add_argument_dockerfile_dir(parser)
    add_argument_distribution_repository_urls(parser)
    add_argument_distribution_repository_key_files(parser)
    add_argument_sourcepkg_dir(parser)
    args = parser.parse_args(argv)

    data = copy.deepcopy(args.__dict__)
    data.update({
        'distribution_repository_urls': args.distribution_repository_urls,
        'distribution_repository_keys': get_distribution_repository_keys(
            args.distribution_repository_urls,
            args.distribution_repository_key_files),
        'target_repository': None,

        'uid': get_user_id(),
        'build_environment_variables': [],
    })
    create_dockerfile(
        'release/rpm/sourcepkg_task.Dockerfile.em', data, args.dockerfile_dir)

    with open(os.path.join(args.dockerfile_dir, 'mock_config.cfg'), 'w') as mock_cfg:
        mock_cfg.write(expand_template('release/rpm/mock_config.cfg.em', data))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# The MIT License (MIT)
# Copyright (c) 2016 Markus Broecker <mbrckr@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import random

def dieRoller(dice, face, drm, count, showStats, hasZero=True):
	''' Rolls a lot of dice and presents the result.
		Dice: the number of dice to roll
		face: the number of faces of each die (eg 6, 10, 20, ....)
		drm: the final DRM applied to the result
		count: how many die rolls should be made
		showStats: whether the resulting distribution should be printed
		hasZero: does the die have a '0' on its face? if not the die range is [1..face]
		returns a list with tuples in the form of (result, count). this result
		is _not_ normalized!
	'''

	highEnd = int(dice * face * 1.5)

	trimLeft = False
	trimRight = True

	def makeRoll():
		if hasZero:
			return random.randint(0, face-1)
		else:
			return random.randint(1, face);

	rolls = []
	for i in range(0, highEnd):
		rolls.append(0)

	for i in range(0,count):
		roll = 0
		for d in range(0, dice):
			roll += makeRoll()

		roll += drm
		roll = max(0, roll)

		try:
			rolls[roll] += 1
		except IndexError:
			rolls.append(1)

	maxRoll = 0
	minRoll = count
	meanRoll = 0
	for r in rolls:
		maxRoll = max(maxRoll, r)
		minRoll = min(minRoll, r)
		meanRoll += r

	# trim the right side of the list
	if trimRight:
		while rolls[-1] == 0:
			rolls.pop()

	# trim the left side of the list
	offset = 0
	if trimLeft:
		while rolls[0] == 0:
			rolls.pop(0)
			offset += 1

	if showStats:
		print(count, 'rolls of %dd' % dice + '%d' % face + ' with DRM %+d' % drm)
		for i in range(0, len(rolls)):

			s = "%4d" % rolls[i] + ' '

			c = round(float(rolls[i]) / maxRoll * 10)
			s += c * '*'

			if rolls[i] == maxRoll:
				s += ' <- Peak'

			print("%02d " % (i + offset), s)

	result = []
	for i in range(0, len(rolls)):
		result.append((i + offset, rolls[i]))

	return result

if __name__ == '__main__':

	drm = 0
	dice = 2
	face = 10
	count = 10000

	dieRoller(dice, face, drm, count, True)
<commit_msg>Fixed bug in die roller<commit_after>#!/usr/bin/env python3

# The MIT License (MIT)
# Copyright (c) 2016 Markus Broecker <mbrckr@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import random

def dieRoller(dice, face, drm, count, showStats, hasZero=True):
	''' Rolls a lot of dice and presents the result.
		Dice: the number of dice to roll
		face: the number of faces of each die (eg 6, 10, 20, ....)
		drm: the final DRM applied to the result
		count: how many die rolls should be made
		showStats: whether the resulting distribution should be printed
		hasZero: does the die have a '0' on its face? if not the die range is [1..face]
		returns a list with tuples in the form of (result, count). this result
		is _not_ normalized!
	'''

	highEnd = int(dice * face * 1.5)

	trimLeft = False
	trimRight = True

	def makeRoll():
		if hasZero:
			return random.randint(0, face-1)
		else:
			return random.randint(1, face);

	rolls = []
	for i in range(0, highEnd):
		rolls.append(0)

	for i in range(0,count):
		roll = 0
		for d in range(0, dice):
			roll += makeRoll()

		roll += drm
		roll = max(0, roll)

		try:
			rolls[roll] += 1
		except IndexError:
			rolls.append(1)

	maxRoll = 0
	minRoll = count
	meanRoll = 0
	for r in rolls:
		maxRoll = max(maxRoll, r)
		minRoll = min(minRoll, r)
		meanRoll += r

	# trim the right side of the list
	if trimRight:
		while rolls[-1] == 0:
			rolls.pop()

	# trim the left side of the list
	offset = 0
	if trimLeft:
		while rolls[0] == 0:
			rolls.pop(0)
			offset += 1

	if showStats:
		print(count, 'rolls of %dd' % dice + '%d' % face + ' with DRM %+d' % drm)
		for i in range(0, len(rolls)):

			s = "%4d" % rolls[i] + ' '

			c = int(round(float(rolls[i]) / maxRoll * 10)
			s += c * '*'

			if rolls[i] == maxRoll:
				s += ' <- Peak'

			print("%02d " % (i + offset), s)

	result = []
	for i in range(0, len(rolls)):
		result.append((i + offset, rolls[i]))

	return result

if __name__ == '__main__':

	drm = 0
	dice = 2
	face = 10
	count = 10000

	dieRoller(dice, face, drm, count, True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Convert delly output to BED format.
"""

import os
import os.path as op
import sys
import logging

from jcvi.formats.base import BaseFile, read_until, must_open
from jcvi.utils.aws import push_to_s3
from jcvi.apps.base import OptionParser, ActionDispatcher, sh, mkdir, need_update

class DelLine (object):

    def __init__(self, line):
        args = line.strip().split("\t")
        self.seqid = args[0]
        self.start = int(args[1]) + 1
        self.end = int(args[2])
        self.size = int(args[3])
        assert self.size == self.end - self.start + 1
        self.supporting_pairs = int(args[4])
        self.avg_mapping_quality = float(args[5])
        self.accn = args[6]

    @property
    def bedline(self):
        return "\t".join(str(x) for x in (self.seqid,
                          self.start - 1, self.end, self.accn,
                          self.supporting_pairs, "+"))

class Delly (BaseFile):

    def __init__(self, filename):
        super(Delly, self).__init__(filename)

    def __iter__(self):
        fp = must_open(self.filename)
        while True:
            read_until(fp, "-----")
            nextline = fp.readline()
            nextline = fp.readline()
            if not nextline.strip():
                break
            d = DelLine(nextline)
            yield d

    def write_bed(self, bedfile="stdout"):
        fw = must_open(bedfile, "w")
        for d in self:
            print >> fw, d.bedline
        logging.debug("File written to `{0}`.".format(bedfile))

def main():

    actions = (
        ('bed', 'Convert del.txt to del.bed'),
        ('mito', 'Find mito deletions in BAM'),
        ('mitocompile', 'Compile mito deletions from multiple VCF files'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

def bed(args):
    """
    %prog bed del.txt

    Convert `del.txt` to BED format. DELLY manual here:
    <http://www.embl.de/~rausch/delly.html>

    Deletion:
    chr, start, end, size, #supporting_pairs, avg._mapping_quality, deletion_id
    chr1, 10180, 10509, 329, 75, 15.8667, Deletion_Sample_00000000
    """
    p = OptionParser(bed.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    delt, = args
    dt = Delly(delt)
    dt.write_bed("del.bed")

def mitocompile(args):
    """
    %prog mitcompile *.vcf.gz

    Extract information about deletions in vcf file.
    """
    from jcvi.formats.vcf import VcfLine
    from urlparse import parse_qsl

    p = OptionParser(mitocompile.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    vcfs = args
    print "\t".join("vcf seqid pos alt svlen pe sr".split())
    for vcf in vcfs:
        fp = must_open(vcf)
        for row in fp:
            if row[0] == '#':
                continue
            v = VcfLine(row)
            info = dict(parse_qsl(v.info))
            print "\t".join(str(x) for x in (vcf, v.seqid, v.pos, v.alt,
                        info.get("SVLEN"), info["PE"], info["SR"]))

def mito(args):
    """
    %prog mito chrM.fa input.bam

    Identify mitochondrial deletions.
    """
    p = OptionParser(mito.__doc__)
    p.set_aws_opts(store="hli-mv-data-science/htang/mito-deletions")
    p.add_option("--realignonly", default=False, action="store_true",
                 help="Realign only")
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    chrMfa, bamfile = args
    store = opts.output_path
    cleanup = not opts.nocleanup

    if not bamfile.endswith(".bam"):
        bamfiles = [x.strip() for x in open(bamfile)]
    else:
        bamfiles = [bamfile]

    for bamfile in bamfiles:
        run_mito(chrMfa, bamfile, opts,
                 realignonly=opts.realignonly,
                 store=store, cleanup=cleanup)

def run_mito(chrMfa, bamfile, opts, realignonly=False, store=None, cleanup=False):
    from jcvi.formats.sam import get_minibam
    region = "chrM"
    minibam = op.basename(bamfile).replace(".bam", ".{}.bam".format(region))
    if not op.exists(minibam):
        get_minibam(bamfile, region)
    else:
        logging.debug("{} found. Skipped.".format(minibam))

    realign = minibam.rsplit(".", 1)[0] + ".realign"
    realignbam = realign + ".bam"
    margs = " -v -t {} -o {}".format(opts.cpus, realign)
    if need_update(minibam, realign + ".bam"):
        cmd = "speedseq realign"
        cmd += margs
        cmd += " {} {}".format(chrMfa, minibam)
        sh(cmd)
    else:
        logging.debug("{} found. Skipped.".format(realignbam))

    if realignonly:
        return

    vcffile = realign + ".sv.vcf.gz"
    if need_update(realignbam, vcffile):
        cmd = "speedseq sv"
        cmd += margs
        cmd += " -R {}".format(chrMfa)
        cmd += " -B {} -D {} -S {}".format(realignbam,
                        realign + ".discordants.bam", realign + ".splitters.bam")
        sh(cmd)
    else:
        logging.debug("{} found. Skipped.".format(vcffile))

    if cleanup:
        sh("rm -f {}* {}*".format(minibam, realignbam))

    if store:
        push_to_s3(store, vcffile)

if __name__ == '__main__':
    main()
<commit_msg>[variation] add piledriver in delly.mito()<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Convert delly output to BED format.
"""

import os.path as op
import sys
import logging

from jcvi.formats.base import BaseFile, read_until, must_open
from jcvi.utils.aws import ls_s3, push_to_s3
from jcvi.apps.base import OptionParser, ActionDispatcher, sh, need_update

class DelLine (object):

    def __init__(self, line):
        args = line.strip().split("\t")
        self.seqid = args[0]
        self.start = int(args[1]) + 1
        self.end = int(args[2])
        self.size = int(args[3])
        assert self.size == self.end - self.start + 1
        self.supporting_pairs = int(args[4])
        self.avg_mapping_quality = float(args[5])
        self.accn = args[6]

    @property
    def bedline(self):
        return "\t".join(str(x) for x in (self.seqid,
                          self.start - 1, self.end, self.accn,
                          self.supporting_pairs, "+"))

class Delly (BaseFile):

    def __init__(self, filename):
        super(Delly, self).__init__(filename)

    def __iter__(self):
        fp = must_open(self.filename)
        while True:
            read_until(fp, "-----")
            nextline = fp.readline()
            nextline = fp.readline()
            if not nextline.strip():
                break
            d = DelLine(nextline)
            yield d

    def write_bed(self, bedfile="stdout"):
        fw = must_open(bedfile, "w")
        for d in self:
            print >> fw, d.bedline
        logging.debug("File written to `{0}`.".format(bedfile))

def main():

    actions = (
        ('bed', 'Convert del.txt to del.bed'),
        ('mito', 'Find mito deletions in BAM'),
        ('mitocompile', 'Compile mito deletions from multiple VCF files'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

def bed(args):
    """
    %prog bed del.txt

    Convert `del.txt` to BED format. DELLY manual here:
    <http://www.embl.de/~rausch/delly.html>

    Deletion:
    chr, start, end, size, #supporting_pairs, avg._mapping_quality, deletion_id
    chr1, 10180, 10509, 329, 75, 15.8667, Deletion_Sample_00000000
    """
    p = OptionParser(bed.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    delt, = args
    dt = Delly(delt)
    dt.write_bed("del.bed")

def mitocompile(args):
    """
    %prog mitcompile *.vcf.gz

    Extract information about deletions in vcf file.
    """
    from jcvi.formats.vcf import VcfLine
    from urlparse import parse_qsl

    p = OptionParser(mitocompile.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    vcfs = args
    print "\t".join("vcf seqid pos alt svlen pe sr".split())
    for vcf in vcfs:
        fp = must_open(vcf)
        for row in fp:
            if row[0] == '#':
                continue
            v = VcfLine(row)
            info = dict(parse_qsl(v.info))
            print "\t".join(str(x) for x in (vcf, v.seqid, v.pos, v.alt,
                        info.get("SVLEN"), info["PE"], info["SR"]))

def mito(args):
    """
    %prog mito chrM.fa input.bam

    Identify mitochondrial deletions.
    """
    p = OptionParser(mito.__doc__)
    p.set_aws_opts(store="hli-mv-data-science/htang/mito-deletions")
    p.add_option("--realignonly", default=False, action="store_true",
                 help="Realign only")
    p.add_option("--svonly", default=False, action="store_true",
                 help="Run Realign => SV calls only")
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    chrMfa, bamfile = args
    store = opts.output_path
    cleanup = not opts.nocleanup

    if not op.exists(chrMfa):
        logging.debug("File `{}` missing. Exiting.".format(chrMfa))

    chrMfai = chrMfa + ".fai"
    if not op.exists(chrMfai):
        cmd = "samtools index {}".format(chrMfa)
        sh(cmd)

    if not bamfile.endswith(".bam"):
        bamfiles = [x.strip() for x in open(bamfile)]
    else:
        bamfiles = [bamfile]

    if store:
        computed = ls_s3(store)
        computed = [op.basename(x).split('.')[0] for x in computed if \
                        x.endswith(".sv.vcf.gz")]
        remaining_samples = [x for x in bamfiles \
                    if op.basename(x).split(".")[0] not in computed]

        logging.debug("Already computed on `{}`: {}".\
                        format(store, len(bamfiles) - len(remaining_samples)))
        bamfiles = remaining_samples

    logging.debug("Total samples: {}".format(len(bamfiles)))

    for bamfile in bamfiles:
        run_mito(chrMfa, bamfile, opts,
                 realignonly=opts.realignonly,
                 svonly=opts.svonly,
                 store=store, cleanup=cleanup)

def run_mito(chrMfa, bamfile, opts, realignonly=False, svonly=False,
             store=None, cleanup=False):
    from jcvi.formats.sam import get_minibam
    region = "chrM"
    minibam = op.basename(bamfile).replace(".bam", ".{}.bam".format(region))
    if not op.exists(minibam):
        get_minibam(bamfile, region)
    else:
        logging.debug("{} found. Skipped.".format(minibam))

    realign = minibam.rsplit(".", 1)[0] + ".realign"
    realignbam = realign + ".bam"
    margs = " -v -t {} -o {}".format(opts.cpus, realign)
    if need_update(minibam, realign + ".bam"):
        cmd = "speedseq realign"
        cmd += margs
        cmd += " {} {}".format(chrMfa, minibam)
        sh(cmd)
    else:
        logging.debug("{} found. Skipped.".format(realignbam))

    if realignonly:
        return

    vcffile = realign + ".sv.vcf.gz"
    if need_update(realignbam, vcffile):
        cmd = "speedseq sv"
        cmd += margs
        cmd += " -R {}".format(chrMfa)
        cmd += " -B {} -D {} -S {}".format(realignbam,
                        realign + ".discordants.bam", realign + ".splitters.bam")
        sh(cmd)
    else:
        logging.debug("{} found. Skipped.".format(vcffile))

    if store:
        push_to_s3(store, vcffile)

    if svonly:
        return

    piledriver = realign + ".piledriver"
    if need_update(realignbam, piledriver):
        cmd = "bamtools piledriver -fasta {}".format(chrMfa)
        cmd += " -in {}".format(realignbam)
        sh(cmd, outfile=piledriver)

    if cleanup:
        sh("rm -f {}* {}*".format(minibam, realignbam))

    if store:
        push_to_s3(store, piledriver)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from helium_client import Helium

helium = Helium("/dev/serial0")
helium.connect()

channel = helium.create_channel("Helium Cloud MQTT")
channel.send("hello from Python")
<commit_msg>Change default channel name<commit_after>from helium_client import Helium

helium = Helium("/dev/serial0")
helium.connect()

channel = helium.create_channel("Helium MQTT")
channel.send("hello from Python")
<|endoftext|>


<commit_before>"""
Execute like this:

$ python bin/generate_test_list.py
tests = [
    'sympy.concrete.tests',
    'sympy.core.tests',
    'sympy.functions.combinatorial.tests',
    'sympy.functions.elementary.tests',
    'sympy.functions.special.tests',
    'sympy.geometry.tests',
    'sympy.integrals.tests',
    'sympy.matrices.tests',
    'sympy.ntheory.tests',
    'sympy.numerics.tests',
    'sympy.parsing.tests',
    'sympy.physics.tests',
    'sympy.plotting.tests',
    'sympy.polynomials.tests',
    'sympy.printing.tests',
    'sympy.series.tests',
    'sympy.simplify.tests',
    'sympy.solvers.tests',
    'sympy.specfun.tests',
    'sympy.test_external',
    'sympy.utilities.tests',
    ]

"""

from __future__ import print_function

from glob import glob

def get_paths(level=15):
    """
    Generates a set of paths for testfiles searching.

    Examples
    ========

    >>> get_paths(2)
    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py']
    >>> get_paths(6)
    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py',
    'sympy/*/*/*/test_*.py', 'sympy/*/*/*/*/test_*.py',
    'sympy/*/*/*/*/*/test_*.py', 'sympy/*/*/*/*/*/*/test_*.py']

    """
    wildcards = ["/"]
    for i in range(level):
        wildcards.append(wildcards[-1] + "*/")
    p = ["sympy" + x + "test_*.py" for x in wildcards]
    return p

g = []
for x in get_paths():
    g.extend(glob(x))
g = [".".join(x.split("/")[:-1]) for x in g]
g = list(set(g))
g.sort()
print("tests = [")
for x in g:
    print("    '%s'," % x)
print("    ]")
<commit_msg>Don't add whitespace to the front of the ] in generate_test_list.py<commit_after>"""
Execute like this:

$ python bin/generate_test_list.py
tests = [
    'sympy.concrete.tests',
    'sympy.core.tests',
    'sympy.functions.combinatorial.tests',
    'sympy.functions.elementary.tests',
    'sympy.functions.special.tests',
    'sympy.geometry.tests',
    'sympy.integrals.tests',
    'sympy.matrices.tests',
    'sympy.ntheory.tests',
    'sympy.numerics.tests',
    'sympy.parsing.tests',
    'sympy.physics.tests',
    'sympy.plotting.tests',
    'sympy.polynomials.tests',
    'sympy.printing.tests',
    'sympy.series.tests',
    'sympy.simplify.tests',
    'sympy.solvers.tests',
    'sympy.specfun.tests',
    'sympy.test_external',
    'sympy.utilities.tests',
    ]

"""

from __future__ import print_function

from glob import glob

def get_paths(level=15):
    """
    Generates a set of paths for testfiles searching.

    Examples
    ========

    >>> get_paths(2)
    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py']
    >>> get_paths(6)
    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py',
    'sympy/*/*/*/test_*.py', 'sympy/*/*/*/*/test_*.py',
    'sympy/*/*/*/*/*/test_*.py', 'sympy/*/*/*/*/*/*/test_*.py']

    """
    wildcards = ["/"]
    for i in range(level):
        wildcards.append(wildcards[-1] + "*/")
    p = ["sympy" + x + "test_*.py" for x in wildcards]
    return p

g = []
for x in get_paths():
    g.extend(glob(x))
g = [".".join(x.split("/")[:-1]) for x in g]
g = list(set(g))
g.sort()
print("tests = [")
for x in g:
    print("    '%s'," % x)
print("]")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Contact Readers
~~~~~~~~~~~~~~~

Reader classes convert input data of various flavors into a chunked stream of
binned contacts.

"""
from __future__ import division, print_function
from collections import OrderedDict, Counter
from contextlib import contextmanager
from bisect import bisect_left
import subprocess
import json
import sys
import six

from pandas.algos import is_lexsorted
import numpy as np
import pandas
import h5py

from ..util import rlencode, get_binsize

class ContactReader(object):
    """
    Interface of a contact reader.

    """
    def size(self):
        """ Total number of contacts """
        raise NotImplementedError

    def __iter__(self):
        """ Iterator over chunks of binned contacts

        Chunks are expected to have the following format:

        * dict of 1D arrays
        * keys `bin1_id`, `bin2_id`, `count`
        * arrays lexically sorted by `bin_id` then `bin2_id`

        """
        raise NotImplementedError

class HDF5Aggregator(ContactReader):
    """
    Aggregate contacts from a hiclib-style HDF5 contacts file.

    """
    def __init__(self, h5pairs, chromsizes, bins, chunksize):
        self.h5 = h5pairs
        self.bins = bins
        self.binsize = get_binsize(bins)
        self.n_contacts = len(self.h5['chrms1'])
        self.n_bins = len(bins)
        self.chunksize = chunksize
        # convert genomic coords of bin starts to absolute
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(chromsizes)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']
        # chrom offset index: chrom_id -> offset in bins
        chrom_nbins =  bins.groupby(bin_chrom_ids, sort=False).size()
        self.chrom_offset = np.r_[0, np.cumsum(chrom_nbins)]
        # index extents of chromosomes on first axis of contact list
        self.partition = self._index_chroms()

    def _index_chroms(self):
        starts, lengths, values = rlencode(self.h5['chrms1'], self.chunksize)
        if len(set(values)) != len(values):
            raise ValueError("Read pair coordinates are not sorted on the first axis")
        return dict(zip(values, zip(starts, starts + lengths)))

    def _load_chunk(self, lo, hi):
        data = OrderedDict([
            ('chrom_id1', self.h5['chrms1'][lo:hi]),
            ('cut1', self.h5['cuts1'][lo:hi]),
            ('chrom_id2', self.h5['chrms2'][lo:hi]),
            ('cut2', self.h5['cuts2'][lo:hi]),
        ])
        return pandas.DataFrame(data)

    def _iterchunks(self, chrom):
        h5pairs = self.h5
        bins = self.bins
        binsize = self.binsize
        chunksize = self.chunksize
        chrom_offset = self.chrom_offset
        cumul_length = self.cumul_length
        abs_start_coords = self.abs_start_coords

        cid = self.idmap[chrom]
        chrom_lo, chrom_hi = self.partition.get(cid, (-1, -1))
        lo = chrom_lo
        hi = lo
        while hi < chrom_hi:
            # fetch next chunk, making sure our selection doesn't split a bin1
            lo, hi = hi, min(hi + chunksize, chrom_hi)
            abs_pos = cumul_length[cid] + h5pairs['cuts1'][hi-1]
            i = int(np.searchsorted(abs_start_coords, abs_pos, side='right')) - 1
            bin_end = bins['end'][i]
            hi = bisect_left(h5pairs['cuts1'], bin_end, lo, chrom_hi)
            print(lo, hi)  # flush=True

            # assign bins to reads
            table = self._load_chunk(lo, hi)
            abs_pos1 = cumul_length[h5pairs['chrms1'][lo:hi]] + h5pairs['cuts1'][lo:hi]
            abs_pos2 = cumul_length[h5pairs['chrms2'][lo:hi]] + h5pairs['cuts2'][lo:hi]
            if np.any(abs_pos1 > abs_pos2):
                raise ValueError(
                    "Found a read pair that maps to the lower triangle of the contact map (side1 > side2). "
                    "Check that the provided chromsome ordering and read pair file are consistent "
                    "such that all pairs map to the upper triangle with respect to the given "
                    "chromosome ordering.")

            if binsize is None:
                table['bin1_id'] = np.searchsorted(abs_start_coords, abs_pos1, side='right') - 1
                table['bin2_id'] = np.searchsorted(abs_start_coords, abs_pos2, side='right') - 1
            else:
                rel_bin1 = np.floor(table['cut1']/binsize).astype(int)
                rel_bin2 = np.floor(table['cut2']/binsize).astype(int)
                table['bin1_id'] = chrom_offset[table['chrom_id1'].values] + rel_bin1
                table['bin2_id'] = chrom_offset[table['chrom_id2'].values] + rel_bin2

            # reduce
            gby = table.groupby(['bin1_id', 'bin2_id'])
            agg = (gby['chrom_id1'].count()
                                   .reset_index()
                                   .rename(columns={'chrom_id1': 'count'}))
            yield {k:v.values for k,v in six.iteritems(agg)}

    def size(self):
        return len(self.h5['chrms1'])

    def __iter__(self):
        for chrom in self.idmap.keys():
            for chunk in self._iterchunks(chrom):
                yield chunk

class TabixAggregator(ContactReader):
    """
    Aggregate contacts from a sorted, BGZIP-compressed and tabix-indexed
    tab-delimited text file.

    """
    def __init__(self, filepath, chromsizes, bins):
        try:
            import pysam
        except ImportError:
            raise ImportError("pysam is required to read tabix files")
        n_bins = len(bins)
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        self.bins = bins
        self.binsize = get_binsize(bins)
        self.pairsfile = pysam.TabixFile(filepath, 'r')
        self.parser = pysam.asTuple()
        # number of lines in file
        p1 = subprocess.Popen(['unpigz',  '-p', '8',  '-c', filepath], stdout=subprocess.PIPE)
        p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
        self.n_records = int(p2.communicate()[0])
        # convert genomic coords of bin starts to absolute
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(chromsizes)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']
        # chrom offset index: chrom_id -> offset in bins
        chrom_nbins =  bins.groupby(bin_chrom_ids, sort=False).size()
        self.chrom_offset = np.r_[0, np.cumsum(chrom_nbins)]

    def _iterchunks(self, chrom):
        pairsfile = self.pairsfile
        chrom_offset = self.chrom_offset
        idmap = self.idmap
        parser = self.parser
        binsize = self.binsize
        bins = self.bins[self.bins.chrom==chrom]

        print(chrom)  # flush=True

        for bin1_id, bin in bins.iterrows():
            accumulator = Counter()
            for record in pairsfile.fetch(chrom, bin.start, bin.end, parser=parser):
                chrom2 = record[3]
                pos2 = int(record[4])
                bin2_id = chrom_offset[idmap[chrom2]] + pos2 // binsize
                accumulator[bin2_id] += 1
            if len(accumulator) == 0:
                continue

            agg = pandas.DataFrame({
                'bin1_id': bin1_id,
                'bin2_id': list(accumulator.keys()),
                'count': list(accumulator.values()),
            }, columns=['bin1_id', 'bin2_id', 'count'])
            agg = agg.sort_values('bin2_id')

            yield {k: v.values for k,v in six.iteritems(agg)}

    def size(self):
        return self.n_records

    def __iter__(self):
        for chrom in self.idmap.keys():
            for chunk in self._iterchunks(chrom):
                yield chunk

class CoolerAggregator(ContactReader):
    """
    Aggregate contacts from an existing Cooler file.

    """
    def __init__(self, cool, bins, chunksize):
        self.cool = cool
        self.new_bins = bins
        self.new_binsize = get_binsize(bins)
        self.old_binsize = self.cool.info['bin-size']

        assert self.new_binsize % self.old_binsize == 0
        self.factor = self.new_binsize // self.old_binsize

        self.chunksize = chunksize
        self.bin1_offset = self.cool._get_index('bin1_offset')

        # convert genomic coords of bin starts to absolute
        chroms = self.cool.chroms()['name'][:].values
        lengths = self.cool.chroms()['length'][:].values
        self.idmap = pandas.Series(index=chroms, data=range(len(chroms)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(lengths)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']

        # chrom offset index: chrom_id -> offset in bins
        self.chrom_offset = np.r_[0, np.cumsum(bins.groupby('chrom', sort=False).size())]

    def _aggregate(self, chunk):
        # use the "start" point as anchor for re-binning
        # XXX - alternatives: midpoint anchor, proportional re-binning
        bins = self.new_bins
        binsize = self.new_binsize
        chrom_offset = self.chrom_offset
        cumul_length = self.cumul_length
        abs_start_coords = self.abs_start_coords

        chrom_id1 = self.idmap.loc[chunk['chrom1'].values]
        chrom_id2 = self.idmap.loc[chunk['chrom2'].values]
        if binsize is None:
            abs_start1 = cumul_length[chrom_id1] + chunk['start1'].values
            abs_start2 = cumul_length[chrom_id2] + chunk['start2'].values
            chunk['bin1_id'] = np.searchsorted(abs_start_coords, abs_start1, side='right') - 1
            chunk['bin2_id'] = np.searchsorted(abs_start_coords, abs_start2, side='right') - 1
        else:
            rel_bin1 = np.floor(chunk['start1']/binsize).astype(int)
            rel_bin2 = np.floor(chunk['start2']/binsize).astype(int)
            chunk['bin1_id'] = chrom_offset[chrom_id1] + rel_bin1
            chunk['bin2_id'] = chrom_offset[chrom_id2] + rel_bin2

        gby = chunk.groupby(['bin1_id', 'bin2_id'], sort=False)
        agg = gby['count'].sum().reset_index()
        return {k: v.values for k,v in six.iteritems(agg)}

    def size(self):
        return self.cool.info['nnz']

    def _iterchunks(self, chrom):
        bin1_offset = self.bin1_offset
        chunksize = self.chunksize
        factor = self.factor

        # it's important to extract some multiple of `factor` rows at a time
        c0, c1 = self.cool.extent(chrom)
        step = (chunksize // factor) * factor
        edges = np.arange(bin1_offset[c0], bin1_offset[c1]+step, step)
        edges[-1] = bin1_offset[c1]

        table = self.cool.pixels(join=True)
        for lo, hi in zip(edges[:-1], edges[1:]):
            chunk = table[lo:hi]
            print(lo, hi)
            yield self._aggregate(chunk)

    def __iter__(self):
        for chrom in self.idmap.keys():
            print(chrom)
            for chunk in self._iterchunks(chrom):
                yield chunk

class SparseLoader(ContactReader):
    """
    Load binned contacts from a single 3-column sparse matrix text file.

    """
    def __init__(self, filepath, chunksize):
        # number of lines in file
        p1 = subprocess.Popen(['unpigz',  '-p', '8',  '-c', filepath], stdout=subprocess.PIPE)
        p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
        self.n_records = int(p2.communicate()[0])

        # file iterator
        self.reader = pandas.read_csv(filepath, sep='\t', iterator=True,
            names=['bin1_id', 'bin2_id', 'count'])
        self.chunksize = chunksize

    def size(self):
        return self.n_records

    def __iter__(self):
        while True:
            try:
                data = self.reader.read(self.chunksize)
                yield {k: v.values for k,v in six.iteritems(data)}
            except StopIteration:
                break

class SparseTileLoader(ContactReader):
    """
    Load binned contacts from a collection of 3-column sparse matrix files
    representing contig-contig contact matrix tiles.

    """
    pass

class DenseLoader(ContactReader):
    """
    Load a dense genome-wide numpy array contact matrix.

    """
    def __init__(self, heatmap):
        # TRIU sparsify the matrix
        i, j = np.nonzero(heatmap)
        mask = i <= j
        triu_i, triu_j = i[mask], j[mask]
        self.data = {
            'bin1_id': triu_i,
            'bin2_id': triu_j,
            'count': heatmap[triu_i, triu_j],
        }
        self.nnz = len(triu_i)

    def size(self):
        return self.nnz

    def __iter__(self):
        yield self.data

class DenseTileLoader(ContactReader):
    """
    Load a contact matrix from a collection of dense numpy array contig-contig
    tiles.

    """
    pass

<commit_msg>Warn when contig is missing from contact list when loading. Fixes #12<commit_after># -*- coding: utf-8 -*-
"""
Contact Readers
~~~~~~~~~~~~~~~

Reader classes convert input data of various flavors into a chunked stream of
binned contacts.

"""
from __future__ import division, print_function
from collections import OrderedDict, Counter
from contextlib import contextmanager
from bisect import bisect_left
import subprocess
import warnings
import json
import sys
import six

from pandas.algos import is_lexsorted
import numpy as np
import pandas
import h5py

from ..util import rlencode, get_binsize

class ContactReader(object):
    """
    Interface of a contact reader.

    """
    def size(self):
        """ Total number of contacts """
        raise NotImplementedError

    def __iter__(self):
        """ Iterator over chunks of binned contacts

        Chunks are expected to have the following format:

        * dict of 1D arrays
        * keys `bin1_id`, `bin2_id`, `count`
        * arrays lexically sorted by `bin_id` then `bin2_id`

        """
        raise NotImplementedError

class HDF5Aggregator(ContactReader):
    """
    Aggregate contacts from a hiclib-style HDF5 contacts file.

    """
    def __init__(self, h5pairs, chromsizes, bins, chunksize):
        self.h5 = h5pairs
        self.bins = bins
        self.binsize = get_binsize(bins)
        self.n_contacts = len(self.h5['chrms1'])
        self.n_bins = len(bins)
        self.chunksize = chunksize
        # convert genomic coords of bin starts to absolute
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(chromsizes)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']
        # chrom offset index: chrom_id -> offset in bins
        chrom_nbins =  bins.groupby(bin_chrom_ids, sort=False).size()
        self.chrom_offset = np.r_[0, np.cumsum(chrom_nbins)]
        # index extents of chromosomes on first axis of contact list
        self.partition = self._index_chroms()

    def _index_chroms(self):
        starts, lengths, values = rlencode(self.h5['chrms1'], self.chunksize)
        if len(set(values)) != len(values):
            raise ValueError("Read pair coordinates are not sorted on the first axis")
        return dict(zip(values, zip(starts, starts + lengths)))

    def _load_chunk(self, lo, hi):
        data = OrderedDict([
            ('chrom_id1', self.h5['chrms1'][lo:hi]),
            ('cut1', self.h5['cuts1'][lo:hi]),
            ('chrom_id2', self.h5['chrms2'][lo:hi]),
            ('cut2', self.h5['cuts2'][lo:hi]),
        ])
        return pandas.DataFrame(data)

    def _iterchunks(self, chrom):
        h5pairs = self.h5
        bins = self.bins
        binsize = self.binsize
        chunksize = self.chunksize
        chrom_offset = self.chrom_offset
        cumul_length = self.cumul_length
        abs_start_coords = self.abs_start_coords

        cid = self.idmap[chrom]
        chrom_lo, chrom_hi = self.partition.get(cid, (-1, -1))
        lo = chrom_lo
        hi = lo
        while hi < chrom_hi:
            # fetch next chunk, making sure our selection doesn't split a bin1
            lo, hi = hi, min(hi + chunksize, chrom_hi)
            abs_pos = cumul_length[cid] + h5pairs['cuts1'][hi-1]
            i = int(np.searchsorted(abs_start_coords, abs_pos, side='right')) - 1
            bin_end = bins['end'][i]
            hi = bisect_left(h5pairs['cuts1'], bin_end, lo, chrom_hi)
            print(lo, hi)  # flush=True

            # assign bins to reads
            table = self._load_chunk(lo, hi)
            abs_pos1 = cumul_length[h5pairs['chrms1'][lo:hi]] + h5pairs['cuts1'][lo:hi]
            abs_pos2 = cumul_length[h5pairs['chrms2'][lo:hi]] + h5pairs['cuts2'][lo:hi]
            if np.any(abs_pos1 > abs_pos2):
                raise ValueError(
                    "Found a read pair that maps to the lower triangle of the contact map (side1 > side2). "
                    "Check that the provided chromsome ordering and read pair file are consistent "
                    "such that all pairs map to the upper triangle with respect to the given "
                    "chromosome ordering.")

            if binsize is None:
                table['bin1_id'] = np.searchsorted(abs_start_coords, abs_pos1, side='right') - 1
                table['bin2_id'] = np.searchsorted(abs_start_coords, abs_pos2, side='right') - 1
            else:
                rel_bin1 = np.floor(table['cut1']/binsize).astype(int)
                rel_bin2 = np.floor(table['cut2']/binsize).astype(int)
                table['bin1_id'] = chrom_offset[table['chrom_id1'].values] + rel_bin1
                table['bin2_id'] = chrom_offset[table['chrom_id2'].values] + rel_bin2

            # reduce
            gby = table.groupby(['bin1_id', 'bin2_id'])
            agg = (gby['chrom_id1'].count()
                                   .reset_index()
                                   .rename(columns={'chrom_id1': 'count'}))
            yield {k:v.values for k,v in six.iteritems(agg)}

    def size(self):
        return len(self.h5['chrms1'])

    def __iter__(self):
        for chrom in self.idmap.keys():
            for chunk in self._iterchunks(chrom):
                yield chunk

class TabixAggregator(ContactReader):
    """
    Aggregate contacts from a sorted, BGZIP-compressed and tabix-indexed
    tab-delimited text file.

    """
    def __init__(self, filepath, chromsizes, bins):
        try:
            import pysam
        except ImportError:
            raise ImportError("pysam is required to read tabix files")
        n_bins = len(bins)
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        self.bins = bins
        self.binsize = get_binsize(bins)
        self.pairsfile = pysam.TabixFile(filepath, 'r')
        self.parser = pysam.asTuple()
        # number of lines in file
        p1 = subprocess.Popen(['unpigz',  '-p', '8',  '-c', filepath], stdout=subprocess.PIPE)
        p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
        self.n_records = int(p2.communicate()[0])
        # convert genomic coords of bin starts to absolute
        self.idmap = pandas.Series(index=chromsizes.keys(), data=range(len(chromsizes)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(chromsizes)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']
        # chrom offset index: chrom_id -> offset in bins
        chrom_nbins =  bins.groupby(bin_chrom_ids, sort=False).size()
        self.chrom_offset = np.r_[0, np.cumsum(chrom_nbins)]

    def _iterchunks(self, chrom):
        pairsfile = self.pairsfile
        chrom_offset = self.chrom_offset
        idmap = self.idmap
        parser = self.parser
        binsize = self.binsize
        bins = self.bins[self.bins.chrom==chrom]

        print(chrom)  # flush=True

        for bin1_id, bin in bins.iterrows():
            accumulator = Counter()
            for record in pairsfile.fetch(chrom, bin.start, bin.end, parser=parser):
                chrom2 = record[3]
                pos2 = int(record[4])
                bin2_id = chrom_offset[idmap[chrom2]] + pos2 // binsize
                accumulator[bin2_id] += 1
            if len(accumulator) == 0:
                continue

            agg = pandas.DataFrame({
                'bin1_id': bin1_id,
                'bin2_id': list(accumulator.keys()),
                'count': list(accumulator.values()),
            }, columns=['bin1_id', 'bin2_id', 'count'])
            agg = agg.sort_values('bin2_id')

            yield {k: v.values for k,v in six.iteritems(agg)}

    def size(self):
        return self.n_records

    def __iter__(self):
        file_contigs = [c.decode('utf-8') for c in self.pairsfile.contigs]
        for chrom in self.idmap.keys():
            if chrom in file_contigs:
                for chunk in self._iterchunks(chrom):
                    yield chunk
            else:
                warnings.warn("Did not find contig '{}' in contact list file.".format(chrom))

class CoolerAggregator(ContactReader):
    """
    Aggregate contacts from an existing Cooler file.

    """
    def __init__(self, cool, bins, chunksize):
        self.cool = cool
        self.new_bins = bins
        self.new_binsize = get_binsize(bins)
        self.old_binsize = self.cool.info['bin-size']

        assert self.new_binsize % self.old_binsize == 0
        self.factor = self.new_binsize // self.old_binsize

        self.chunksize = chunksize
        self.bin1_offset = self.cool._get_index('bin1_offset')

        # convert genomic coords of bin starts to absolute
        chroms = self.cool.chroms()['name'][:].values
        lengths = self.cool.chroms()['length'][:].values
        self.idmap = pandas.Series(index=chroms, data=range(len(chroms)))
        bin_chrom_ids = self.idmap[bins['chrom']].values
        self.cumul_length = np.r_[0, np.cumsum(lengths)]
        self.abs_start_coords = self.cumul_length[bin_chrom_ids] + bins['start']

        # chrom offset index: chrom_id -> offset in bins
        self.chrom_offset = np.r_[0, np.cumsum(bins.groupby('chrom', sort=False).size())]

    def _aggregate(self, chunk):
        # use the "start" point as anchor for re-binning
        # XXX - alternatives: midpoint anchor, proportional re-binning
        bins = self.new_bins
        binsize = self.new_binsize
        chrom_offset = self.chrom_offset
        cumul_length = self.cumul_length
        abs_start_coords = self.abs_start_coords

        chrom_id1 = self.idmap.loc[chunk['chrom1'].values]
        chrom_id2 = self.idmap.loc[chunk['chrom2'].values]
        if binsize is None:
            abs_start1 = cumul_length[chrom_id1] + chunk['start1'].values
            abs_start2 = cumul_length[chrom_id2] + chunk['start2'].values
            chunk['bin1_id'] = np.searchsorted(abs_start_coords, abs_start1, side='right') - 1
            chunk['bin2_id'] = np.searchsorted(abs_start_coords, abs_start2, side='right') - 1
        else:
            rel_bin1 = np.floor(chunk['start1']/binsize).astype(int)
            rel_bin2 = np.floor(chunk['start2']/binsize).astype(int)
            chunk['bin1_id'] = chrom_offset[chrom_id1] + rel_bin1
            chunk['bin2_id'] = chrom_offset[chrom_id2] + rel_bin2

        gby = chunk.groupby(['bin1_id', 'bin2_id'], sort=False)
        agg = gby['count'].sum().reset_index()
        return {k: v.values for k,v in six.iteritems(agg)}

    def size(self):
        return self.cool.info['nnz']

    def _iterchunks(self, chrom):
        bin1_offset = self.bin1_offset
        chunksize = self.chunksize
        factor = self.factor

        # it's important to extract some multiple of `factor` rows at a time
        c0, c1 = self.cool.extent(chrom)
        step = (chunksize // factor) * factor
        edges = np.arange(bin1_offset[c0], bin1_offset[c1]+step, step)
        edges[-1] = bin1_offset[c1]

        table = self.cool.pixels(join=True)
        for lo, hi in zip(edges[:-1], edges[1:]):
            chunk = table[lo:hi]
            print(lo, hi)
            yield self._aggregate(chunk)

    def __iter__(self):
        for chrom in self.idmap.keys():
            print(chrom)
            for chunk in self._iterchunks(chrom):
                yield chunk

class SparseLoader(ContactReader):
    """
    Load binned contacts from a single 3-column sparse matrix text file.

    """
    def __init__(self, filepath, chunksize):
        # number of lines in file
        p1 = subprocess.Popen(['unpigz',  '-p', '8',  '-c', filepath], stdout=subprocess.PIPE)
        p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
        self.n_records = int(p2.communicate()[0])

        # file iterator
        self.reader = pandas.read_csv(filepath, sep='\t', iterator=True,
            names=['bin1_id', 'bin2_id', 'count'])
        self.chunksize = chunksize

    def size(self):
        return self.n_records

    def __iter__(self):
        while True:
            try:
                data = self.reader.read(self.chunksize)
                yield {k: v.values for k,v in six.iteritems(data)}
            except StopIteration:
                break

class SparseTileLoader(ContactReader):
    """
    Load binned contacts from a collection of 3-column sparse matrix files
    representing contig-contig contact matrix tiles.

    """
    pass

class DenseLoader(ContactReader):
    """
    Load a dense genome-wide numpy array contact matrix.

    """
    def __init__(self, heatmap):
        # TRIU sparsify the matrix
        i, j = np.nonzero(heatmap)
        mask = i <= j
        triu_i, triu_j = i[mask], j[mask]
        self.data = {
            'bin1_id': triu_i,
            'bin2_id': triu_j,
            'count': heatmap[triu_i, triu_j],
        }
        self.nnz = len(triu_i)

    def size(self):
        return self.nnz

    def __iter__(self):
        yield self.data

class DenseTileLoader(ContactReader):
    """
    Load a contact matrix from a collection of dense numpy array contig-contig
    tiles.

    """
    pass

<|endoftext|>


<commit_before>#!/usr/bin/env python

from matplotlib import pyplot as plt
from scipy.special import erfinv
from math import sqrt, pi, exp, cos, sin, erf, ceil

K = 10000
p = 0.1
p0 = 0.001
w = int(ceil(2*sqrt(p*K)*erfinv(1-p0)))
print 'max ops: %d, band radius: %d' % (K, w)

radius = lambda x: int(ceil(2*sqrt(p*x)*erfinv(1-p0)))
xs = [x*100 for x in range(100)]
ys = [radius(x) for x in xs]
plt.plot(xs, ys, '-k')
plt.grid(True)
plt.savefig('radius.png', dpi=300)
raise

nrange = range(-2*w, 2*w +1)
u = {}
cos_rl = lambda n: ((n+0.5)*pi)/w
sin_rl = lambda n: n*pi/2
fourier_cos = lambda x, t, n: (2*(-1)**n / ((n+0.5)*pi)) * exp(t*-p*cos_rl(n)**2) * cos(cos_rl(n)*x)
fourier_sin = lambda x, t, n: (2*(-1)**(n+1) / (n*pi))   * exp(t*-p*sin_rl(n)**2) * sin(sin_rl(n)*x) if n else 0
diffusion_correct = lambda N, x, t: sum(fourier_cos(x,t,n) + fourier_sin(x,t,n) for n in range(N))
diffusion_nobc = lambda x: 0.5 * ( erf((x+w)/(2*sqrt(p*K))) - erf((x-w)/(2*sqrt(p*K))) )

for k in range(K+1):
    if k == 0:
        u[k] = {n: 1 if abs(n) < w + K else 0 for n in nrange}
        continue
    u[k] = {}
    for n in nrange:
        if abs(n) >= w:
            u[k][n] = 0
        else:
            u[k][n] = p*u[k-1][n-1] + p*u[k-1][n+1] + (1-2*p)*u[k-1][n]

plt.rc('text', usetex=1)
ticks_every = 20
plt.xticks([x * ticks_every + ticks_every*(nrange[0]/ticks_every) for x in range(len(nrange)/ticks_every+1)], rotation='vertical', fontsize=10)
plt.yticks([0.1*n for n in range(11)], fontsize=10)
plt.scatter(nrange, [u[K][n] for n in nrange], color='k', marker='o', s=5, label='recurrence solution')
plt.grid(True)
plt.ylim(-0.1, 1.1)
plt.xlabel('$n$')
plt.ylabel('$u(n,K)$')
plt.title('band randius = %d for $p_\circ=%.4f$ after $K=%d$ steps' % (w, p0, K), size=10)

num_terms = 4
plt.plot(nrange, [diffusion_correct(num_terms, n, K) if abs(n) <= w else 0 for n in nrange],
    color='greenyellow', label='%d terms of F.S.: $u(0, K)=%.4f$' % (num_terms*2-1, diffusion_correct(4, 0, K)))
plt.plot(nrange, [diffusion_nobc(n) for n in nrange], '--', color='r', label='ignoring B.C: $\hat{u}(0, K)=%.4f$' % diffusion_nobc(0))
plt.legend(fontsize=8)
plt.savefig('diffusion.png', dpi=300)
<commit_msg>refactor diffusion.py<commit_after>#!/usr/bin/env python

from matplotlib import pyplot as plt
from scipy.special import erfinv
from math import sqrt, pi, exp, cos, sin, erf, ceil

K = 10000
p = 0.1
p0 = 0.001
w = int(ceil(2*sqrt(p*K)*erfinv(1-p0)))
print 'max ops: %d, band radius: %d' % (K, w)

def radii():
    radius = lambda x: int(ceil(2*sqrt(p*x)*erfinv(1-p0)))
    xs = [x*100 for x in range(100)]
    ys = [radius(x) for x in xs]
    return xs, ys

#plt.plot(xs, ys, '-k')
#plt.grid(True)
#plt.savefig('radius.png', dpi=300)

plt.rc('text', usetex=1)
ticks_every = 20
nrange = range(-2*w, 2*w +1)
plt.xticks([x * ticks_every + ticks_every*(nrange[0]/ticks_every) for x in range(len(nrange)/ticks_every+1)], rotation='vertical', fontsize=10)
plt.yticks([0.1*n for n in range(11)], fontsize=10)
plt.grid(True)
plt.ylim(-0.1, 1.1)
plt.xlabel('$n$')
plt.ylabel('$u(n,K)$')
plt.title('band randius = %d for $p_\circ=%.4f$ after $K=%d$ steps' % (w, p0, K), size=10)

def recurrence_solution():
    u = {}
    for k in range(K+1):
        if k == 0:
            u[k] = {n: 1 if abs(n) < w + K else 0 for n in nrange}
            continue
        u[k] = {}
        for n in nrange:
            if abs(n) >= w:
                u[k][n] = 0
            else:
                u[k][n] = p*u[k-1][n-1] + p*u[k-1][n+1] + (1-2*p)*u[k-1][n]
    return u

u = recurrence_solution()
plt.scatter(nrange, [u[K][n] for n in nrange], edgecolor='g', facecolor='none', lw=0.5, s=20, label='recurrence solution')

def fs_solution(num_terms):
    cos_rl = lambda n: ((n+0.5)*pi)/w
    sin_rl = lambda n: n*pi/2
    fourier_cos = lambda x, t, n: (2*(-1)**n / ((n+0.5)*pi)) * exp(t*-p*cos_rl(n)**2) * cos(cos_rl(n)*x)
    fourier_sin = lambda x, t, n: (2*(-1)**(n+1) / (n*pi))   * exp(t*-p*sin_rl(n)**2) * sin(sin_rl(n)*x) if n else 0
    diffusion_correct = lambda N, x, t: sum(fourier_cos(x,t,n) + fourier_sin(x,t,n) for n in range(N))
    return [diffusion_correct(num_terms, n, K) if abs(n) <= w else 0 for n in nrange]

num_terms = 4
soln = fs_solution(num_terms)
plt.plot(nrange, soln, color='k', label='%d terms of F.S.: $u(0, K)=%.4f$' % (num_terms*2-1, soln[nrange.index(0)]))

def nobc_approx_solution():
    return [0.5 * ( erf((n+w)/(2*sqrt(p*K))) - erf((n-w)/(2*sqrt(p*K))) ) for n in nrange]

soln = nobc_approx_solution()
plt.plot(nrange, soln, color='r', label='ignoring B.C: $\hat{u}(0, K)=%.4f$' % soln[nrange.index(0)])
plt.legend(fontsize=8)
plt.savefig('diffusion.png', dpi=300)
<|endoftext|>


<commit_before>__version_info__ = {
    'major': 0,
    'minor': 3,
    'micro': 0,
    'releaselevel': 'beta',
    'serial': 1
}

def get_version(short=False):
    assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final')
    vers = ["%(major)i.%(minor)i.%(micro)i" % __version_info__, ]
    if __version_info__['releaselevel'] != 'final' and not short:
        vers.append('%s%i' % (__version_info__['releaselevel'][0],
                    __version_info__['serial']))
    return ''.join(vers)

__version__ = get_version()
<commit_msg>0.3.0 Release<commit_after>__version_info__ = {
    'major': 0,
    'minor': 3,
    'micro': 0,
    'releaselevel': 'final',
    'serial': 1
}

def get_version(short=False):
    assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final')
    vers = ["%(major)i.%(minor)i.%(micro)i" % __version_info__, ]
    if __version_info__['releaselevel'] != 'final' and not short:
        vers.append('%s%i' % (__version_info__['releaselevel'][0],
                    __version_info__['serial']))
    return ''.join(vers)

__version__ = get_version()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

VERSION = (0, 5, 6)

def get_version():
    version = '%s.%s.%s' % (VERSION[0], VERSION[1], VERSION[2])
    return version

__version__ = get_version()
<commit_msg>version bump<commit_after># -*- coding: utf-8 -*-

VERSION = (0, 5, 7)

def get_version():
    version = '%s.%s.%s' % (VERSION[0], VERSION[1], VERSION[2])
    return version

__version__ = get_version()
<|endoftext|>


<commit_before># Copyright (c) 2014 bivio Software, Inc.  All rights reserved.

import flask
from flask.ext.wtf import Form
import publicprize.auth.model as pam
import publicprize.contest.model as pcm
from publicprize import controller
from wtforms import StringField, TextAreaField
from wtforms.validators import DataRequired

class ContestantForm(Form):
    display_name = StringField('Project Name', validators=[DataRequired()])
    contestant_desc = TextAreaField('Project Summary', validators=[DataRequired()])
    youtube_code = StringField(
        'YouTube Video Code',
        validators=[DataRequired()]
    )
    slideshow_code = StringField(
        'SlideShare Pitch Deck Code',
        validators=[DataRequired()]
    )
    founder_desc = StringField('Founder Short Bio', validators=[DataRequired()])
    website = StringField('Project Website')

    def execute(self, contest):
        if self.is_submitted() and self.validate():
            c = self._update_models(contest)
            if c:
                return flask.redirect(c.format_uri('contestant'))
        return flask.render_template(
            'contest/submit.html',
            contest=contest,
            form=self
        )

    def validate_and_execute(self, contest):
        if self.validate_on_submit():
            return form
        return False

    def _update_models(self, contest):
        c = pcm.Contestant()
        self.populate_obj(c)
        f = pcm.Founder()
        self.populate_obj(f)
        f.display_name = flask.session['user.display_name']
        controller.db.session.add(c)
        controller.db.session.add(f)
        controller.db.session.flush()
        controller.db.session.add(
            pam.BivAccess(
                source_biv_id=contest.biv_id,
                target_biv_id=c.biv_id
            )
        )
        controller.db.session.add(
            pam.BivAccess(
                source_biv_id=c.biv_id,
                target_biv_id=f.biv_id
            )
        )
        return c
<commit_msg>added url validation<commit_after># Copyright (c) 2014 bivio Software, Inc.  All rights reserved.

import flask
from flask.ext.wtf import Form
import publicprize.auth.model as pam
import publicprize.contest.model as pcm
from publicprize import controller
import re
import urllib.request
from wtforms import StringField, TextAreaField
from wtforms.validators import DataRequired

class ContestantForm(Form):
    display_name = StringField('Project Name', validators=[DataRequired()])
    contestant_desc = TextAreaField(
        'Project Summary', validators=[DataRequired()])
    youtube_code = StringField(
        'YouTube Video URL',
        validators=[DataRequired()]
    )
    slideshow_code = StringField(
        'SlideShare Pitch Deck URL',
        validators=[DataRequired()]
    )
    founder_desc = StringField(
        'Founder Short Bio', validators=[DataRequired()])
    website = StringField('Project Website')

    def execute(self, contest):
        if self.is_submitted() and self.validate():
            c = self._update_models(contest)
            if c:
                return flask.redirect(c.format_uri('contestant'))
        return flask.render_template(
            'contest/submit.html',
            contest=contest,
            form=self
        )

    def validate(self):
        if super().validate():
            self._validate_youtube()
            self._validate_slideshare()
            self._validate_website()
            if not self.errors:
                return True
        return False

    def _slideshare_code(self):
        # www.slideshare.net/benjaminevans/culture-kitchen-pitch-deck-18074260
        # www.slideshare.net/slideshow/embed_code/18074260
        value = self.slideshow_code.data
        m = re.search('(\d{5}\d+)$', value)
        if m:
            return m.group(1)
        return None

    def _update_models(self, contest):
        self.youtube_code.data = self._youtube_code()
        self.slideshow_code.data = self._slideshare_code()
        c = pcm.Contestant()
        self.populate_obj(c)
        c.is_public = controller.app().config['PP_ALL_PUBLIC_CONTESTANTS']
        f = pcm.Founder()
        self.populate_obj(f)
        f.display_name = flask.session['user.display_name']
        controller.db.session.add(c)
        controller.db.session.add(f)
        controller.db.session.flush()
        controller.db.session.add(
            pam.BivAccess(
                source_biv_id=contest.biv_id,
                target_biv_id=c.biv_id
            )
        )
        controller.db.session.add(
            pam.BivAccess(
                source_biv_id=c.biv_id,
                target_biv_id=f.biv_id
            )
        )
        return c

    def _youtube_code(self):
        value = self.youtube_code.data
        # http://youtu.be/a1Y73sPHKxw
        # or https://www.youtube.com/watch?v=a1Y73sPHKxw
        if re.search('\?', value) and re.search('v\=', value):
            m = re.search(r'(?:\?|\&)v\=(.*)', value)
            if m:
                return m.group(1)
        else:
            m = re.search(r'\/([^\&\?\/]+)$', value)
            if m:
                return m.group(1)
        return None

    def _validate_url(self, url):
        if not re.search(r'^http', url):
            url = 'http://' + url
        try:
            req = urllib.request.urlopen(url, None, 10)
            req.read()
            req.close()
        except urllib.request.URLError:
            return False
        except ValueError:
            return False
        return True

    def _validate_website(self):
        if self.website.data:
            if not self._validate_url(self.website.data):
                self.website.errors = ['Website invalid or unavailable']

    def _validate_youtube(self):
        code = self._youtube_code()
        if code:
            if not self._validate_url('http://youtu.be/' + code):
                self.youtube_code.errors = [
                    'Unknown YouTube VIDEO_ID: ' + code]
        else:
            self.youtube_code.errors = ['Invalid YouTube URL']

    def _validate_slideshare(self):
        code = self._slideshare_code()
        if code:
            if not self._validate_url(
                'http://www.slideshare.net/slideshow/embed_code/' + code):
                self.slideshow_code.errors = ['Unknown SlideShare ID: ' + code]
        else:
            self.slideshow_code.errors = ['Invalid SlideShare URL']
<|endoftext|>