<commit_before>#
# Stochastic degradation toy model.
#
# This file is part of PINTS.
#  Copyright (c) 2017-2019, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
from __future__ import absolute_import, division
from __future__ import print_function, unicode_literals
import numpy as np
from scipy.interpolate import interp1d
import pints

from . import ToyModel

class StochasticDegradationModel(pints.ForwardModel, ToyModel):

    """
    Stochastic degradation model of a single chemical reaction starting from
    an initial concentration :math:: n0 and degrading to 0 according to the
    following model:
    .. math::
    \\A \rightarrow{\text{k}} 0 [1] \\\\

    The model is simulated according to the Gillespie algorithm [2]:
    1. Sample a random value r from a uniform distribution:
    :math::
    \\r ~ unif(0,1) \\
    2. Calculate the time ($\tau$) until the next single reaction as follows:
    .. math::
    \\ \tau = \frac{1}{A(t)k} * ln{\frac{1}{r}} [1] \\\\
    3. Update the molecule count at time t + .. math:: \\tau as:
    .. math::
    \\A(t + \tau) = A(t)-1 \\\\
    4. Return to step (1) until molecule count reaches 0

    Has one parameter: Rate constant :math:`k`.
    :math:`r` is a random variable, which is part of the stochastic model
    The initial concentration :math:`A(0) = n_0` can be set using the
    (optional) named constructor arg ``initial_concentration``

    [1] A Practical Guide to Stochastic Simulations of Reaction Diffusion
    Processes. Erban, Radek (2007). arXiv:0704.1908 [q-bio.SC]
    [2] A general method for numerically simulating the stochastic time
    evolution of coupled chemical reactions. Gillespie, Daniel (1976).
    Journal of Computational Physics

    *Extends:* :class:`pints.ForwardModel`, :class:`pints.toy.ToyModel`.
    """

    def __init__(self, initial_concentration=20):
        super(StochasticDegradationModel, self).__init__()
        self._n0 = float(initial_concentration)
        if self._n0 < 0:
            raise ValueError('Initial concentration cannot be negative.')

        self._interp_func = None
        self._mol_count = []
        self._time = []

    def n_parameters(self):
        """ See :meth:`pints.ForwardModel.n_parameters()`. """
        return 1

    def simulate(self, parameters, times):
        """ See :meth:`pints.ForwardModel.simulate()`. """
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')
        if self._n0 == 0:
            return np.zeros(times.shape)

        t = 0
        a = self._n0
        self._mol_count = [a]
        self._time = [t]

        # Run stochastic degradation algorithm, calculating time until next
        # reaction and decreasing concentration by 1 at that time
        while a > 0:
            r = np.random.uniform(0, 1)
            t += (1 / (a * k)) * np.log(1 / r)
            self._time.append(t)
            a = a - 1
            self._mol_count.append(a)

        # Interpolate as step function, decreasing mol_count by 1 at each
        # reaction time point
        self._interp_func = interp1d(self._time, self._mol_count,
                                     kind='previous')

        # Compute concentration values at given time points using f1
        # at any time beyond the last reaction, concentration = 0
        values = self._interp_func(times[np.where(times <= max(self._time))])
        zero_vector = np.zeros(len(times[np.where(times > max(self._time))]))
        values = np.concatenate((values, zero_vector))

        return values

    def deterministic_mean(self, parameters, times):
        """ Calculates deterministic mean of infinitely many stochastic
        simulations, which follows :math:: n0*exp(-kt)"""
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')

        mean = self._n0 * np.exp(-k * times)
        return mean

    def deterministic_variance(self, parameters, times):
        """ Calculates deterministic variance of infinitely many stochastic
        simulations, which follows :math:: exp(-2kt)(-1 + exp(kt)) * n0"""
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')

        variance = np.exp(-2 * k * times) * (-1 + np.exp(k * times)) * self._n0
        return variance

    def suggested_parameters(self):
        """ See :meth:`pints.toy.ToyModel.suggested_parameters()`. """
        return np.array([0.1])

    def suggested_times(self):
        """ See "meth:`pints.toy.ToyModel.suggested_times()`."""
        return np.linspace(0, 100, 101)
<commit_msg>updated stochastic degradation model code<commit_after>#
# Stochastic degradation toy model.
#
# This file is part of PINTS.
#  Copyright (c) 2017-2019, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
from __future__ import absolute_import, division
from __future__ import print_function, unicode_literals
import numpy as np
from scipy.interpolate import interp1d
import pints

from . import ToyModel

class StochasticDegradationModel(pints.ForwardModel, ToyModel):

    """
    Stochastic degradation model of a single chemical reaction starting from
    an initial concentration :math:: n0 and degrading to 0 according to the
    following model:
    .. math::
        A \\rightarrow{k} 0 [1] \\\\

    The model is simulated according to the Gillespie algorithm [2]:
    1. Sample a random value r from a uniform distribution:
    :math::
        \\r ~ unif(0,1) \\
    2. Calculate the time ($\tau$) until the next single reaction as follows:
    .. math::
        \\tau &= \\frac{1}{A(t)k} * ln{\frac{1}{r}} [1] \\\\
    3. Update the molecule count at time t + .. math:: \\tau as:
    .. math::
        \\A(t + \\tau) = A(t)-1 \\\\
    4. Return to step (1) until molecule count reaches 0

    Has one parameter: Rate constant :math:`k`.
    :math:`r` is a random variable, which is part of the stochastic model
    The initial concentration :math:`A(0) = n_0` can be set using the
    (optional) named constructor arg ``initial_concentration``

    [1] A Practical Guide to Stochastic Simulations of Reaction Diffusion
    Processes. Erban, Radek (2007). arXiv:0704.1908 [q-bio.SC]
    [2] A general method for numerically simulating the stochastic time
    evolution of coupled chemical reactions. Gillespie, Daniel (1976).
    Journal of Computational Physics

    *Extends:* :class:`pints.ForwardModel`, :class:`pints.toy.ToyModel`.
    """

    def __init__(self, initial_concentration=20):
        super(StochasticDegradationModel, self).__init__()
        self._n0 = float(initial_concentration)
        if self._n0 < 0:
            raise ValueError('Initial concentration cannot be negative.')

        self._interp_func = None
        self._mol_count = []
        self._time = []

    def n_parameters(self):
        """ See :meth:`pints.ForwardModel.n_parameters()`. """
        return 1

    def simulate(self, parameters, times):
        """ See :meth:`pints.ForwardModel.simulate()`. """
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')
        if self._n0 == 0:
            return np.zeros(times.shape)

        t = 0
        a = self._n0
        self._mol_count = [a]
        self._time = [t]

        # Run stochastic degradation algorithm, calculating time until next
        # reaction and decreasing concentration by 1 at that time
        while a > 0:
            r = np.random.uniform(0, 1)
            t += (1 / (a * k)) * np.log(1 / r)
            self._time.append(t)
            a = a - 1
            self._mol_count.append(a)

        # Interpolate as step function, decreasing mol_count by 1 at each
        # reaction time point
        self._interp_func = interp1d(self._time, self._mol_count,
                                     kind='previous')

        # Compute concentration values at given time points using f1
        # at any time beyond the last reaction, concentration = 0
        values = self._interp_func(times[np.where(times <= max(self._time))])
        zero_vector = np.zeros(len(times[np.where(times > max(self._time))]))
        values = np.concatenate((values, zero_vector))

        return values

    def deterministic_mean(self, parameters, times):
        """ Calculates deterministic mean of infinitely many stochastic
        simulations, which follows :math:: n0*exp(-kt)"""
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')

        mean = self._n0 * np.exp(-k * times)
        return mean

    def deterministic_variance(self, parameters, times):
        """ Calculates deterministic variance of infinitely many stochastic
        simulations, which follows :math:: exp(-2kt)(-1 + exp(kt)) * n0"""
        parameters = np.asarray(parameters)
        if len(parameters) != self.n_parameters():
            raise ValueError('This model should have only 1 parameter.')
        k = parameters[0]

        if k <= 0:
            raise ValueError('Rate constant must be positive.')

        times = np.asarray(times)
        if np.any(times < 0):
            raise ValueError('Negative times are not allowed.')

        variance = np.exp(-2 * k * times) * (-1 + np.exp(k * times)) * self._n0
        return variance

    def suggested_parameters(self):
        """ See :meth:`pints.toy.ToyModel.suggested_parameters()`. """
        return np.array([0.1])

    def suggested_times(self):
        """ See "meth:`pints.toy.ToyModel.suggested_times()`."""
        return np.linspace(0, 100, 101)
<|endoftext|>


<commit_before>"""
This script is used to train a given model.
"""
import collections
import keras
import keras.backend as K
import os
import sys
import src.features.build_features as build_features

# Parameters
WINDOW_WIDTH_MS = 30  # How many MS of audio to feed into the MLP at a time
SAMPLING_RATE_HZ = 32000  # Sample the audio at this rate
NUM_CHANNELS = 1  # The number of channels in the audio
NUM_EPOCHS = 10000
BATCH_SIZE = 32
LOG_FILE = "log.csv"

def fscore(pred, label):
    """
    Metric for calculating F1 score. An F1 score is a good way to measure when there is a class imbalance.
    It can be interpreted as a weighted average of precision and recall.

    Precision maxes out when there are few false positives.
    Recall maxes out when there are few false negatives.
    """
    false_negatives = K.sum(K.round(K.clip(label - pred, 0, 1)))
    false_positives = K.sum(K.round(K.clip(pred - label, 0, 1)))
    true_positives = K.sum(K.round(pred * label))
    true_negatives = K.sum(K.round((1 - pred) * (1 - label)))

    pres = true_positives / (true_positives + false_positives + 1E-9)
    rec = true_positives / (true_positives + false_negatives + 1E-9)
    fscore = pres * rec / (pres + rec + 1E-9)
    return fscore

class GraphMetrics(keras.callbacks.Callback):
    def __init__(self, metrics):
        """
        """
        self.batch_num = 0
        self.metrics_logs = collections.OrderedDict({"loss" : []})
        if "accuracy" in metrics:
            self.metrics_logs["acc"] = []
        for m in metrics:
            if hasattr(m, "__name__"):
                self.metrics_logs[m.__name__] = []

        # Make the file
        with open(LOG_FILE, 'w') as f:
            f.write(", ".join([metric for metric in self.metrics_logs.keys()]))

    def on_batch_end(self, batch, logs={}):
        for metric in self.metrics_logs.keys():
            self.metrics_logs[metric].append(float(logs.get(metric)))
        with open(LOG_FILE, 'a') as f:
            f.write(os.linesep)
            f.write("----- " + str(self.batch_num) + " -----" + os.linesep)

            # This is pretty crappy code. There must be a way to do this with zip, but I
            # am too busy to think about it right now.
            metrics = [m for m in self.metrics_logs.values()]
            for i in range(len(metrics[0])):
                values = [m[i] for m in metrics]
                f.write(", ".join([str(v) for v in values]))
                f.write(os.linesep)
            for ls in self.metrics_logs.values():
                ls.clear()

        self.batch_num += 1

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("USAGE: Go to the top level directory and run `make train`")
        exit(1)

    model_dir_path = sys.argv[1]
    data_dir_path = sys.argv[2]
    samples_per_window = int(WINDOW_WIDTH_MS * SAMPLING_RATE_HZ / 1000)
    print("Input dimension:", samples_per_window)

    model = keras.models.Sequential()
    model.add(keras.layers.Dense(4024, input_dim=samples_per_window // 2 + 1))  # Takes the abs of the FFT output
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(1012))
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(512))
    model.add(keras.layers.Dropout(0.10))
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(1))
    model.add(keras.layers.Activation("sigmoid"))

    metrics = ["accuracy", fscore]
    model.compile(optimizer="nadam", loss="binary_crossentropy", metrics=metrics)

    kwargs = {"samples_per_vector": samples_per_window,
              "batch_size": BATCH_SIZE,
              "sampling_frequency_hz": SAMPLING_RATE_HZ,
              "channels": NUM_CHANNELS,
              "ignore": ["debug_NO", "debug_VO", "test_split"],
              "include": None,
             }
    data_generator = build_features.generate_data(data_dir_path, **kwargs)
    steps_per_epoch = build_features.calculate_steps_per_epoch(data_dir_path, **kwargs)
    checkpointer = keras.callbacks.ModelCheckpoint(model_dir_path)
    metrics_grapher = GraphMetrics(metrics)
    model.fit_generator(data_generator, steps_per_epoch=steps_per_epoch, epochs=NUM_EPOCHS,
                        callbacks=[checkpointer, metrics_grapher], verbose=1)

<commit_msg>Fixed FScore<commit_after>"""
This script is used to train a given model.
"""
import collections
import keras
import keras.backend as K
import os
import sys
import src.features.build_features as build_features

# Parameters
WINDOW_WIDTH_MS = 30  # How many MS of audio to feed into the MLP at a time
SAMPLING_RATE_HZ = 32000  # Sample the audio at this rate
NUM_CHANNELS = 1  # The number of channels in the audio
NUM_EPOCHS = 10000
BATCH_SIZE = 32
LOG_FILE = "log.csv"

def fscore(pred, label):
    """
    Metric for calculating F1 score. An F1 score is a good way to measure when there is a class imbalance.
    It can be interpreted as a weighted average of precision and recall.

    Precision maxes out when there are few false positives.
    Recall maxes out when there are few false negatives.
    """
    false_negatives = K.sum(K.round(K.clip(label - pred, 0, 1)))
    false_positives = K.sum(K.round(K.clip(pred - label, 0, 1)))
    true_positives = K.sum(K.round(pred * label))
    true_negatives = K.sum(K.round((1 - pred) * (1 - label)))

    pres = true_positives / (true_positives + false_positives + 1E-9)
    rec = true_positives / (true_positives + false_negatives + 1E-9)
    fscore = 2 * pres * rec / (pres + rec + 1E-9)
    return fscore

class GraphMetrics(keras.callbacks.Callback):
    def __init__(self, metrics):
        """
        """
        self.batch_num = 0
        self.metrics_logs = collections.OrderedDict({"loss" : []})
        if "accuracy" in metrics:
            self.metrics_logs["acc"] = []
        for m in metrics:
            if hasattr(m, "__name__"):
                self.metrics_logs[m.__name__] = []

        # Make the file
        with open(LOG_FILE, 'w') as f:
            f.write(", ".join([metric for metric in self.metrics_logs.keys()]))

    def on_batch_end(self, batch, logs={}):
        for metric in self.metrics_logs.keys():
            self.metrics_logs[metric].append(float(logs.get(metric)))
        with open(LOG_FILE, 'a') as f:
            f.write(os.linesep)
            f.write("----- " + str(self.batch_num) + " -----" + os.linesep)

            # This is pretty crappy code. There must be a way to do this with zip, but I
            # am too busy to think about it right now.
            metrics = [m for m in self.metrics_logs.values()]
            for i in range(len(metrics[0])):
                values = [m[i] for m in metrics]
                f.write(", ".join([str(v) for v in values]))
                f.write(os.linesep)
            for ls in self.metrics_logs.values():
                ls.clear()

        self.batch_num += 1

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("USAGE: Go to the top level directory and run `make train`")
        exit(1)

    model_dir_path = sys.argv[1]
    data_dir_path = sys.argv[2]
    samples_per_window = int(WINDOW_WIDTH_MS * SAMPLING_RATE_HZ / 1000)
    print("Input dimension:", samples_per_window)

    model = keras.models.Sequential()
    model.add(keras.layers.Dense(4024, input_dim=samples_per_window // 2 + 1))  # Takes the abs of the FFT output
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(1012))
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(512))
    model.add(keras.layers.Dropout(0.10))
    model.add(keras.layers.Activation("relu"))
    model.add(keras.layers.Dense(1))
    model.add(keras.layers.Activation("sigmoid"))

    metrics = ["accuracy", fscore]
    model.compile(optimizer="nadam", loss="binary_crossentropy", metrics=metrics)

    kwargs = {"samples_per_vector": samples_per_window,
              "batch_size": BATCH_SIZE,
              "sampling_frequency_hz": SAMPLING_RATE_HZ,
              "channels": NUM_CHANNELS,
              "ignore": ["debug_NO", "debug_VO", "test_split"],
              "include": None,
             }
    data_generator = build_features.generate_data(data_dir_path, **kwargs)
    steps_per_epoch = build_features.calculate_steps_per_epoch(data_dir_path, **kwargs)
    checkpointer = keras.callbacks.ModelCheckpoint(model_dir_path)
    metrics_grapher = GraphMetrics(metrics)
    model.fit_generator(data_generator, steps_per_epoch=steps_per_epoch, epochs=NUM_EPOCHS,
                        callbacks=[checkpointer, metrics_grapher], verbose=1)

<|endoftext|>


<commit_before>import six
from datetime import datetime

import pytz

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat().decode('utf-8')

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        element = ''.join(element)
        try:
            element = six.u(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps
<commit_msg>add formatting function to util<commit_after>import six
from datetime import datetime

import pytz

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat().decode('utf-8')

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        element = ''.join(element)
        try:
            element = six.u(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps

def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')
<|endoftext|>


<commit_before>#!/usr/bin/env python
from optparse import OptionParser
from collections import OrderedDict
import multiprocessing
import os
import sys
import time

import h5py
import numpy as np
import pyBigWig
import pysam

import basenji
from basenji_hdf5 import bigwig_batch

'''
basenji_genes.py

Tile a set of genes and save the result in HDF5 for Basenji processing.

Notes:
 -At the moment, I'm excluding target measurements, but that could be included
  if I want to measure accuracy on specific genes.
'''

################################################################################
# main
################################################################################
def main():
    usage = 'usage: %prog [options] <fasta_file> <gtf_file> <hdf5_file>'
    parser = OptionParser(usage)
    parser.add_option('-g', dest='genome_file', default='%s/assembly/human.hg19.genome'%os.environ['HG19'], help='Chromosome lengths file [Default: %default]')
    parser.add_option('-l', dest='seq_length', default=1024, type='int', help='Sequence length [Default: %default]')
    parser.add_option('-c', dest='center_t', default=1/3, type='float', help='Center proportion in which TSSs are required to be [Default: %default]')
    parser.add_option('-p', dest='processes', default=1, type='int', help='Number parallel processes to load data [Default: %default]')
    parser.add_option('-t', dest='target_wigs_file', default=None, help='Store target values, extracted from this list of WIG files')
    parser.add_option('-w', dest='pool_width', type='int', default=1, help='Average pooling width [Default: %default]')
    (options,args) = parser.parse_args()

    if len(args) != 3:
        parser.error('Must provide genes as GTF, genome FASTA, and output HDF5')
    else:
        fasta_file = args[0]
        gtf_file = args[1]
        hdf5_file = args[2]

    ################################################################
    # organize transcript TSS's by chromosome

    # read transcripts
    transcripts = basenji.gff.read_genes(gtf_file, key_id='transcript_id')

    # read transcript --> gene mapping
    transcript_genes = basenji.gff.t2g(gtf_file)

    # open fasta
    fasta = pysam.Fastafile(fasta_file)

    # hash transcript TSS's by chromosome
    chrom_tss = {}
    for tx_id in transcripts:
        if transcripts[tx_id].chrom in fasta.references:
            chrom_tss.setdefault(transcripts[tx_id].chrom,[]).append((transcripts[tx_id].tss(),tx_id))

    # sort TSS's by chromosome
    for chrom in chrom_tss:
        chrom_tss[chrom].sort()

    ################################################################
    # determine segments / map transcripts

    chrom_sizes = OrderedDict()
    for line in open(options.genome_file):
        a = line.split()
        chrom_sizes[a[0]] = int(a[1])

    merge_distance = options.center_t * options.seq_length

    seq_coords = []
    transcript_map = OrderedDict()

    # ordering by options.genome_file allows for easier
    #  bigwig output in downstream scripts.

    for chrom in chrom_sizes:
        if chrom in chrom_tss:
            ctss = chrom_tss[chrom]

            left_i = 0
            while left_i < len(ctss):
                # left TSS
                left_tss = ctss[left_i][0]

                # right TSS
                right_i = left_i
                while right_i+1 < len(ctss) and ctss[right_i+1][0] - left_tss < merge_distance:
                    right_i += 1
                right_tss = ctss[right_i][0]

                # determine segment midpoint
                seg_mid = (left_tss + right_tss) // 2

                # extend
                seg_start = seg_mid - options.seq_length//2
                seg_end = seg_start + options.seq_length

                # save segment
                if seg_start >= 0 and seg_end < chrom_sizes[chrom]:
                    seq_coords.append((chrom,seg_start,seg_end))

                    # annotate TSS to indexes
                    seg_index = len(seq_coords)-1
                    for i in range(left_i,right_i+1):
                        tx_tss, tx_id = ctss[i]
                        tx_pos = (tx_tss - seg_start) // options.pool_width
                        transcript_map[tx_id] = (seg_index,tx_pos)

                # update
                left_i = right_i + 1

    ################################################################
    # extract target values

    if options.target_wigs_file:
        t0 = time.time()

        # get wig files and labels
        target_wigs = OrderedDict()
        for line in open(options.target_wigs_file):
            a = line.split()
            target_wigs[a[0]] = a[1]

        # initialize multiprocessing pool
        pool = multiprocessing.Pool(options.processes)

        # bigwig_read parameters
        bwt_params = [(wig_file, transcript_map, seq_coords, options.pool_width) for wig_file in target_wigs.values()]

        # pull the target values in parallel
        transcript_targets = pool.starmap(bigwig_transcripts, bwt_params)

        # convert to array
        transcript_targets = np.transpose(np.array(transcript_targets))

    ################################################################
    # extract sequences

    seqs_1hot = []

    for chrom, start, end in seq_coords:
        seq = fasta.fetch(chrom, start, end)
        seqs_1hot.append(basenji.dna_io.dna_1hot(seq))

    seqs_1hot = np.array(seqs_1hot)

    fasta.close()

    ################################################################
    # save to HDF5

    # write to HDF5
    hdf5_out = h5py.File(hdf5_file, 'w')

    # store pooling
    hdf5_out.create_dataset('pool_width', data=options.pool_width, dtype='int')

    # store transcript map
    transcripts = list(transcript_map.keys())
    transcript_index = np.array([transcript_map[tx_id][0] for tx_id in transcripts])
    transcript_pos = np.array([transcript_map[tx_id][1] for tx_id in transcripts])

    hdf5_out.create_dataset('transcripts', data=np.array(transcripts, dtype='S'))
    hdf5_out.create_dataset('transcript_index', data=transcript_index)
    hdf5_out.create_dataset('transcript_pos', data=transcript_pos)

    # store genes
    genes = [transcript_genes[tx_id] for tx_id in transcripts]
    hdf5_out.create_dataset('genes', data=np.array(genes, dtype='S'))

    # store sequences
    hdf5_out.create_dataset('seqs_1hot', data=seqs_1hot, dtype='bool')

    # store targets
    if options.target_wigs_file:
        # labels
        target_labels = np.array([tl for tl in target_wigs.keys()], dtype='S')
        hdf5_out.create_dataset('target_labels', data=target_labels)

        # values
        hdf5_out.create_dataset('transcript_targets', data=transcript_targets, dtype='float16')

    # store sequence coordinates
    seq_chrom = np.array([sc[0] for sc in seq_coords], dtype='S')
    seq_start = np.array([sc[1] for sc in seq_coords])
    seq_end = np.array([sc[2] for sc in seq_coords])

    hdf5_out.create_dataset('seq_chrom', data=seq_chrom)
    hdf5_out.create_dataset('seq_start', data=seq_start)
    hdf5_out.create_dataset('seq_end', data=seq_end)

    hdf5_out.close()

################################################################################
def bigwig_transcripts(wig_file, transcript_map, seq_coords, pool_width=1):
    ''' Read gene target values from a bigwig

    Args:
      wig_file: Bigwig filename
      transcript_map: OrderedDict mapping transcript_id to (seq index,seq pos) tuples
      seq_coords: list of (chrom,start,end) sequence coordinates
      pool_width: average pool adjacent nucleotides of this width

    Returns:
      transcript_targets:
    '''

    # initialize target values
    transcript_targets = np.zeros(len(transcript_map), dtype='float16')

    # open wig
    wig_in = pyBigWig.open(wig_file)

    # so we can warn about missing chromosomes just once
    warned_chroms = set()

    # for each transcript
    tx_i = 0
    for transcript in transcript_map:
        # determine sequence and position
        seq_i, seq_pos = transcript_map[transcript]

        # extract sequence coordinates
        seq_chrom, seq_start, seq_end = seq_coords[seq_i]

        # determine gene genomic coordinates
        tx_start = seq_start + seq_pos*pool_width
        tx_end = tx_start + pool_width

        # pull mean value
        try:
            transcript_targets[tx_i] = wig_in.stats(seq_chrom, tx_start, tx_end)[0]
        except RuntimeError:
            if chrom not in warned_chroms:
                print("WARNING: %s doesn't see %s %s:%d-%d. Setting to all zeros. No additional warnings will be offered for %s" % (wig_file,transcript,seq_chrom,seq_start,seq_end,seq_chrom), file=sys.stderr)
                warned_chroms.add(chrom)

        tx_i += 1

    # close wig file
    wig_in.close()

    return transcript_targets

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<commit_msg>warning bug<commit_after>#!/usr/bin/env python
from optparse import OptionParser
from collections import OrderedDict
import multiprocessing
import os
import sys
import time

import h5py
import numpy as np
import pyBigWig
import pysam

import basenji
from basenji_hdf5 import bigwig_batch

'''
basenji_genes.py

Tile a set of genes and save the result in HDF5 for Basenji processing.

Notes:
 -At the moment, I'm excluding target measurements, but that could be included
  if I want to measure accuracy on specific genes.
'''

################################################################################
# main
################################################################################
def main():
    usage = 'usage: %prog [options] <fasta_file> <gtf_file> <hdf5_file>'
    parser = OptionParser(usage)
    parser.add_option('-g', dest='genome_file', default='%s/assembly/human.hg19.genome'%os.environ['HG19'], help='Chromosome lengths file [Default: %default]')
    parser.add_option('-l', dest='seq_length', default=1024, type='int', help='Sequence length [Default: %default]')
    parser.add_option('-c', dest='center_t', default=1/3, type='float', help='Center proportion in which TSSs are required to be [Default: %default]')
    parser.add_option('-p', dest='processes', default=1, type='int', help='Number parallel processes to load data [Default: %default]')
    parser.add_option('-t', dest='target_wigs_file', default=None, help='Store target values, extracted from this list of WIG files')
    parser.add_option('-w', dest='pool_width', type='int', default=1, help='Average pooling width [Default: %default]')
    (options,args) = parser.parse_args()

    if len(args) != 3:
        parser.error('Must provide genes as GTF, genome FASTA, and output HDF5')
    else:
        fasta_file = args[0]
        gtf_file = args[1]
        hdf5_file = args[2]

    ################################################################
    # organize transcript TSS's by chromosome

    # read transcripts
    transcripts = basenji.gff.read_genes(gtf_file, key_id='transcript_id')

    # read transcript --> gene mapping
    transcript_genes = basenji.gff.t2g(gtf_file)

    # open fasta
    fasta = pysam.Fastafile(fasta_file)

    # hash transcript TSS's by chromosome
    chrom_tss = {}
    for tx_id in transcripts:
        if transcripts[tx_id].chrom in fasta.references:
            chrom_tss.setdefault(transcripts[tx_id].chrom,[]).append((transcripts[tx_id].tss(),tx_id))

    # sort TSS's by chromosome
    for chrom in chrom_tss:
        chrom_tss[chrom].sort()

    ################################################################
    # determine segments / map transcripts

    chrom_sizes = OrderedDict()
    for line in open(options.genome_file):
        a = line.split()
        chrom_sizes[a[0]] = int(a[1])

    merge_distance = options.center_t * options.seq_length

    seq_coords = []
    transcript_map = OrderedDict()

    # ordering by options.genome_file allows for easier
    #  bigwig output in downstream scripts.

    for chrom in chrom_sizes:
        if chrom in chrom_tss:
            ctss = chrom_tss[chrom]

            left_i = 0
            while left_i < len(ctss):
                # left TSS
                left_tss = ctss[left_i][0]

                # right TSS
                right_i = left_i
                while right_i+1 < len(ctss) and ctss[right_i+1][0] - left_tss < merge_distance:
                    right_i += 1
                right_tss = ctss[right_i][0]

                # determine segment midpoint
                seg_mid = (left_tss + right_tss) // 2

                # extend
                seg_start = seg_mid - options.seq_length//2
                seg_end = seg_start + options.seq_length

                # save segment
                if seg_start >= 0 and seg_end < chrom_sizes[chrom]:
                    seq_coords.append((chrom,seg_start,seg_end))

                    # annotate TSS to indexes
                    seg_index = len(seq_coords)-1
                    for i in range(left_i,right_i+1):
                        tx_tss, tx_id = ctss[i]
                        tx_pos = (tx_tss - seg_start) // options.pool_width
                        transcript_map[tx_id] = (seg_index,tx_pos)

                # update
                left_i = right_i + 1

    ################################################################
    # extract target values

    if options.target_wigs_file:
        t0 = time.time()

        # get wig files and labels
        target_wigs = OrderedDict()
        for line in open(options.target_wigs_file):
            a = line.split()
            target_wigs[a[0]] = a[1]

        # initialize multiprocessing pool
        pool = multiprocessing.Pool(options.processes)

        # bigwig_read parameters
        bwt_params = [(wig_file, transcript_map, seq_coords, options.pool_width) for wig_file in target_wigs.values()]

        # pull the target values in parallel
        transcript_targets = pool.starmap(bigwig_transcripts, bwt_params)

        # convert to array
        transcript_targets = np.transpose(np.array(transcript_targets))

    ################################################################
    # extract sequences

    seqs_1hot = []

    for chrom, start, end in seq_coords:
        seq = fasta.fetch(chrom, start, end)
        seqs_1hot.append(basenji.dna_io.dna_1hot(seq))

    seqs_1hot = np.array(seqs_1hot)

    fasta.close()

    ################################################################
    # save to HDF5

    # write to HDF5
    hdf5_out = h5py.File(hdf5_file, 'w')

    # store pooling
    hdf5_out.create_dataset('pool_width', data=options.pool_width, dtype='int')

    # store transcript map
    transcripts = list(transcript_map.keys())
    transcript_index = np.array([transcript_map[tx_id][0] for tx_id in transcripts])
    transcript_pos = np.array([transcript_map[tx_id][1] for tx_id in transcripts])

    hdf5_out.create_dataset('transcripts', data=np.array(transcripts, dtype='S'))
    hdf5_out.create_dataset('transcript_index', data=transcript_index)
    hdf5_out.create_dataset('transcript_pos', data=transcript_pos)

    # store genes
    genes = [transcript_genes[tx_id] for tx_id in transcripts]
    hdf5_out.create_dataset('genes', data=np.array(genes, dtype='S'))

    # store sequences
    hdf5_out.create_dataset('seqs_1hot', data=seqs_1hot, dtype='bool')

    # store targets
    if options.target_wigs_file:
        # labels
        target_labels = np.array([tl for tl in target_wigs.keys()], dtype='S')
        hdf5_out.create_dataset('target_labels', data=target_labels)

        # values
        hdf5_out.create_dataset('transcript_targets', data=transcript_targets, dtype='float16')

    # store sequence coordinates
    seq_chrom = np.array([sc[0] for sc in seq_coords], dtype='S')
    seq_start = np.array([sc[1] for sc in seq_coords])
    seq_end = np.array([sc[2] for sc in seq_coords])

    hdf5_out.create_dataset('seq_chrom', data=seq_chrom)
    hdf5_out.create_dataset('seq_start', data=seq_start)
    hdf5_out.create_dataset('seq_end', data=seq_end)

    hdf5_out.close()

################################################################################
def bigwig_transcripts(wig_file, transcript_map, seq_coords, pool_width=1):
    ''' Read gene target values from a bigwig

    Args:
      wig_file: Bigwig filename
      transcript_map: OrderedDict mapping transcript_id to (seq index,seq pos) tuples
      seq_coords: list of (chrom,start,end) sequence coordinates
      pool_width: average pool adjacent nucleotides of this width

    Returns:
      transcript_targets:
    '''

    # initialize target values
    transcript_targets = np.zeros(len(transcript_map), dtype='float16')

    # open wig
    wig_in = pyBigWig.open(wig_file)

    # so we can warn about missing chromosomes just once
    warned_chroms = set()

    # for each transcript
    tx_i = 0
    for transcript in transcript_map:
        # determine sequence and position
        seq_i, seq_pos = transcript_map[transcript]

        # extract sequence coordinates
        seq_chrom, seq_start, seq_end = seq_coords[seq_i]

        # determine gene genomic coordinates
        tx_start = seq_start + seq_pos*pool_width
        tx_end = tx_start + pool_width

        # pull mean value
        try:
            transcript_targets[tx_i] = wig_in.stats(seq_chrom, tx_start, tx_end)[0]
        except RuntimeError:
            if seq_chrom not in warned_chroms:
                print("WARNING: %s doesn't see %s %s:%d-%d. Setting to all zeros. No additional warnings will be offered for %s" % (wig_file,transcript,seq_chrom,seq_start,seq_end,seq_chrom), file=sys.stderr)
                warned_chroms.add(chrom)

        tx_i += 1

    # close wig file
    wig_in.close()

    return transcript_targets

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2017 IBM RESEARCH. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
"""Test for the QASM parser"""

import unittest

import qiskit.qasm as Qasm

from .common import QiskitTestCase

def parse(file_path, prec=15):
    """
      Simple helper
      - file_path: Path to the OpenQASM file
      - prec: Precision for the returned string
    """
    qasm = Qasm.Qasm(file_path)
    return qasm.parse().qasm(prec)

class TestParser(QiskitTestCase):
    """QasmParser"""
    def setUp(self):
        self.QASM_FILE_PATH = self._get_resource_path('qasm/example.qasm')
        self.QASM_FILE_PATH_FAIL = self._get_resource_path(
            'qasm/example_fail.qasm')

    def test_parser(self):
        """should return a correct response for a valid circuit."""

        res = parse(self.QASM_FILE_PATH)
        # TODO: For now only some basic checks.
        self.assertEqual(len(res), 1660)
        self.assertEqual(res[:12], "OPENQASM 2.0")
        self.assertEqual(res[14:41], "gate u3(theta,phi,lambda) q")
        self.assertEqual(res[1644:1659], "measure r -> d;")

    def test_parser_fail(self):
        """should fail a for a  not valid circuit."""

        self.assertRaisesRegex(Qasm.QasmError, "Perhaps there is a missing",
                               parse, file_path=self.QASM_FILE_PATH_FAIL)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add parser tests for all AST item being Nodes<commit_after># -*- coding: utf-8 -*-

# Copyright 2017 IBM RESEARCH. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
"""Test for the QASM parser"""

import unittest

from qiskit.qasm import Qasm, QasmError
from qiskit.qasm._node._node import Node

from .common import QiskitTestCase

def parse(file_path, prec=15):
    """
      Simple helper
      - file_path: Path to the OpenQASM file
      - prec: Precision for the returned string
    """
    qasm = Qasm(file_path)
    return qasm.parse().qasm(prec)

class TestParser(QiskitTestCase):
    """QasmParser"""
    def setUp(self):
        self.QASM_FILE_PATH = self._get_resource_path('qasm/example.qasm')
        self.QASM_FILE_PATH_FAIL = self._get_resource_path(
            'qasm/example_fail.qasm')

    def test_parser(self):
        """should return a correct response for a valid circuit."""

        res = parse(self.QASM_FILE_PATH)
        # TODO: For now only some basic checks.
        self.assertEqual(len(res), 1660)
        self.assertEqual(res[:12], "OPENQASM 2.0")
        self.assertEqual(res[14:41], "gate u3(theta,phi,lambda) q")
        self.assertEqual(res[1644:1659], "measure r -> d;")

    def test_parser_fail(self):
        """should fail a for a  not valid circuit."""

        self.assertRaisesRegex(QasmError, "Perhaps there is a missing",
                               parse, file_path=self.QASM_FILE_PATH_FAIL)

    def test_all_valid_nodes(self):
        """Test that the tree contains only Node subclasses."""
        def inspect(node):
            for child in node.children:
                self.assertTrue(isinstance(child, Node))
                inspect(child)

        qasm = Qasm(self.QASM_FILE_PATH)
        res = qasm.parse()
        inspect(res)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from .OptomechDevice import OptomechDevice
from .Device import Device

class FilterSet(Device, OptomechDevice):
    """Represents an optical filter or filter set.

    This device class has two main purposes:  (1) Providing access to metadata
    about filters in the optical path between a light source and the sample.
    (2) Providing access to metadata about filters in the optical path between
    an imaging device and the sample.

    FilterSets may be simple pass-through optics like an ND filter placed in front of
    a light source, or they may have multiple beam paths like in a dichroic filter cube.

    For swappable filters, see the FilterWheel class.

    Configuration examples::

        Blue_Filter:
            Driver: "FilterSet"
            name: "Blue"
            description: "A combination blue bandpass and ND filter"
            optics:
                # order of filters always starts with closest-to-sample
                0:
                    model: "Semrock FF01-474/27"
                    passBands: ([460, 490])
                1:
                    model: "Unknown ND filter"
                    transmission: 0.13

        EGFP_FilterCube:
            driver: "FilterSet"
            name: "EGFP"
            description: "Epifluorescence filter cube for EGFP (blue excitation / green emission)"
            ports:  # filter cube has two ports for attaching children: excitation and emission

                # note: order of filters always starts with closest-to-sample, regardless of
                # whether this is an excitation or emission path
                excitation:
                    0:
                        model: "Semrock FF506-Di03 (reflected)"
                        passBands: [(None, 505*nm)]  # reflects wavelengths < 505 nm
                    1:
                        model: "Semrock FF02-485/20"
                        passBands: [(473*nm, 498*nm)]  # transmits 473-498 nm
                emission:
                    0:
                        model: "Semrock FF506-Di03 (transmitted)"
                        passBands: [(505, None)]  # transmits wavelengths > 505 nm
                    1:
                        model: "Semrock FF01-524/24"
                        passBands: [(510*nm, 540*nm)]  # transmits 510-540 nm

    """
    def __init__(self, dm, config, name):
        Device.__init__(self, dm, config, name)

        self.config = config

        # build config for optomechdevice
        omconfig = config.copy()
        optics = omconfig.pop('optics', {})
        ports = omconfig.pop('ports', {})
        if len(optics) > 0:
            assert 'default' not in ports, "Config cannot have both `optics` and `ports: default`"
            ports['default'] = optics

        omconfig['ports'] = list(ports.keys())
        omconfig['optics'] = ports

        OptomechDevice.__init__(self, dm, config, name)

    def description(self):
        return self._config.get('description')
<commit_msg>fixup filterset interpretation of config formats<commit_after>from .OptomechDevice import OptomechDevice
from .Device import Device

class FilterSet(Device, OptomechDevice):
    """Represents an optical filter or filter set.

    This device class has two main purposes:  (1) Providing access to metadata
    about filters in the optical path between a light source and the sample.
    (2) Providing access to metadata about filters in the optical path between
    an imaging device and the sample.

    FilterSets may be simple pass-through optics like an ND filter placed in front of
    a light source, or they may have multiple beam paths like in a dichroic filter cube.

    For swappable filters, see the FilterWheel class.

    Configuration examples::

        Blue_Filter:
            Driver: "FilterSet"
            name: "Blue"
            description: "A combination blue bandpass and ND filter"
            optics:
                # order of filters always starts with closest-to-sample
                0:
                    model: "Semrock FF01-474/27"
                    passBands: ([460, 490])
                1:
                    model: "Unknown ND filter"
                    transmission: 0.13

        EGFP_FilterCube:
            driver: "FilterSet"
            name: "EGFP"
            description: "Epifluorescence filter cube for EGFP (blue excitation / green emission)"
            ports:  # filter cube has two ports for attaching children: excitation and emission

                # note: order of filters always starts with closest-to-sample, regardless of
                # whether this is an excitation or emission path
                excitation:
                    0:
                        model: "Semrock FF506-Di03 (reflected)"
                        passBands: [(None, 505*nm)]  # reflects wavelengths < 505 nm
                    1:
                        model: "Semrock FF02-485/20"
                        passBands: [(473*nm, 498*nm)]  # transmits 473-498 nm
                emission:
                    0:
                        model: "Semrock FF506-Di03 (transmitted)"
                        passBands: [(505, None)]  # transmits wavelengths > 505 nm
                    1:
                        model: "Semrock FF01-524/24"
                        passBands: [(510*nm, 540*nm)]  # transmits 510-540 nm

    """
    def __init__(self, dm, config, name):
        Device.__init__(self, dm, config, name)

        self.config = config

        # build config for optomechdevice
        omconfig = config.copy()
        optics = omconfig.pop('optics', {})
        ports = omconfig.pop('ports', {})
        if len(optics) > 0:
            assert 'default' not in ports, "Config cannot have both `optics` and `ports: default`"
            ports['default'] = optics

        omconfig['ports'] = list(ports.keys())

        # Ports can either be specified like:
        #    {'first_port': [{optic1}, {optic2}, ...], 'second_port': [...]}
        # or like:
        #    {'first_port': {0: {optic1}, 1: {optic2}, ...}, 'second_port': {...}}
        # (the second form is easier to write out in config files)
        for p, optics in ports.items():
            if isinstance(optics, list):
                continue
            elif isinstance(optics, dict):
                # convert {0:x, 1:y} dict to list
                keys = sorted(list(optics.keys()))
                assert len(set(keys)) == int(keys[-1]) + 1, "Optics dict must have contiguous integer keys; got: %r" % keys
                ports[p] = [optics[k] for k in keys]
            else:
                raise TypeError("Optics must be described by a list or a dict; got: %r" % optics)

        omconfig['optics'] = ports

        OptomechDevice.__init__(self, dm, omconfig, name)

    def description(self):
        return self._config.get('description')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import logging
import tweepy
import re
import requests
import report
from time import time
from bot import Bot

logger = logging.getLogger(__name__)

class TwitterDMListener(Bot):

    def get_api(self, user):
        keys = user.get_twitter_credentials()
        auth = tweepy.OAuthHandler(consumer_key=keys[0],
                                   consumer_secret=keys[1])
        auth.set_access_token(keys[2],  # access_token_key
                              keys[3])  # access_token_secret
        return tweepy.API(auth, wait_on_rate_limit=True)

    def crawl(self, user):
        """
        crawls all Tweets which mention the bot from the twitter rest API.

        :return: reports: (list of report.Report objects)
        """
        reports = []
        try:
            if user.get_last_twitter_request() + 60 > time():
                return reports
        except TypeError:
            user.set_last_twitter_request(time())
        try:
            api = self.get_api(user)
        except TypeError:
            return reports  # no twitter account for this user.
        last_dm = user.get_seen_dm()
        try:
            if last_dm is None:
                mentions = api.direct_messages()
            else:
                mentions = api.direct_messages(since_id=last_dm[0])
            user.set_last_twitter_request(time())
            for status in mentions:
                text = re.sub(
                       "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
                       "", status.text)
                reports.append(report.Report(status.author.screen_name,
                                             "twitterDM",
                                             text,
                                             status.id,
                                             status.created_at))
            user.save_seen_dm(last_dm)
            return reports
        except tweepy.RateLimitError:
            logger.error("Twitter API Error: Rate Limit Exceeded",
                         exc_info=True)
            # :todo implement rate limiting
        except requests.exceptions.ConnectionError:
            logger.error("Twitter API Error: Bad Connection", exc_info=True)
        except tweepy.TweepError as terror:
            # Waiting for https://github.com/tweepy/tweepy/pull/1109 to get
            # merged, so direct messages work again
            if terror.api_code == 34:
                return reports
            logger.error("Twitter API Error: General Error", exc_info=True)
        return reports

    def post(self, user, report):
        pass
<commit_msg>twitter DMs don't work anyway<commit_after><|endoftext|>


<commit_before>from pyshelf.search.metadata import Metadata
from pyshelf.search import utils
import yaml
import os

class ElasticInitializer(object):
    def __init__(self, config_path):
        self.config_path = config_path

    def initialize(self):
        config = self.read_config()
        wrapper = utils.configure_es_connection(config["connectionString"],
                config.get("accessKey"), config.get("secretKey"), config.get("region"))
        Metadata.init(using=wrapper.connection, index=wrapper.index)
        self.es.indices.refresh(index=index)

    def read_config(self):
        with open(self.config_path) as cf:
            config = yaml.load(cf.read())

            return config.get("elasticsearch")

bin_dir = os.path.dirname(os.path.realpath(__file__))
config_path = os.path.realpath(os.path.join(bin_dir, "../config.yaml"))
elastic = ElasticInitializer(config_path)
elastic.initialize()
<commit_msg>One more<commit_after>from pyshelf.search.metadata import Metadata
from pyshelf.search import utils
import yaml
import os

class ElasticInitializer(object):
    def __init__(self, config_path):
        self.config_path = config_path

    def initialize(self):
        config = self.read_config()
        wrapper = utils.configure_es_connection(config["connectionString"],
                config.get("accessKey"), config.get("secretKey"), config.get("region"))
        Metadata.init(using=wrapper.connection, index=wrapper.index)
        self.es.indices.refresh(index=wrapper.index)

    def read_config(self):
        with open(self.config_path) as cf:
            config = yaml.load(cf.read())

            return config.get("elasticsearch")

bin_dir = os.path.dirname(os.path.realpath(__file__))
config_path = os.path.realpath(os.path.join(bin_dir, "../config.yaml"))
elastic = ElasticInitializer(config_path)
elastic.initialize()
<|endoftext|>


<commit_before># This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2016 ScyllaDB

import os
import re
from abc import abstractmethod, ABCMeta
import time
import logging
from typing import NamedTuple

from sdcm.prometheus import NemesisMetrics
from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms

LOGGER = logging.getLogger(__name__)

class MetricsPosition(NamedTuple):
    ops: int
    lat_mean: int
    lat_med: int
    lat_perc_95: int
    lat_perc_99: int
    lat_perc_999: int
    lat_max: int
    errors: int

class StressExporter(FileFollowerThread, metaclass=ABCMeta):
    METRICS_GAUGES = {}

    def __init__(self, instance_name: str, metrics: NemesisMetrics, stress_operation: str, stress_log_filename: str,
                 loader_idx: int, cpu_idx: int = 1):  # pylint: disable=too-many-arguments

        super(StressExporter, self).__init__()
        self.metrics = metrics
        self.stress_operation = stress_operation
        self.stress_log_filename = stress_log_filename
        gauge_name = self.create_metrix_gauge()
        self.stress_metric = self.METRICS_GAUGES[gauge_name]
        self.instance_name = instance_name
        self.loader_idx = loader_idx
        self.cpu_idx = cpu_idx
        self.metrics_positions = self.merics_position_in_log()
        self.keyspace = ''

    @abstractmethod
    def merics_position_in_log(self) -> MetricsPosition:
        ...

    @abstractmethod
    def create_metrix_gauge(self) -> str:
        ...

    def set_metric(self, name: str, value: float) -> None:
        self.stress_metric.labels(0, self.instance_name, self.loader_idx, self.cpu_idx, name, self.keyspace).set(value)

    def clear_metrics(self) -> None:
        if self.stress_metric:
            for metric_name in [metric for metric in self.metrics_positions._fields]:
                self.set_metric(metric_name, 0.0)

    @staticmethod
    @abstractmethod
    def skip_line(line: str) -> bool:
        ...

    @staticmethod
    @abstractmethod
    def split_line(line: str) -> list:
        ...

    def get_metric_value(self, columns: list, metric_name: str) -> str:
        try:
            value = columns[getattr(self.metrics_positions, metric_name)]
        except (ValueError, IndexError) as exc:
            value = ''
            LOGGER.warning("Faile to get %s metric value. Error: %s", metric_name, str(exc))

        return value

    def run(self):
        while not self.stopped():
            exists = os.path.isfile(self.stress_log_filename)
            if not exists:
                time.sleep(0.5)
                continue

            for line in self.follow_file(self.stress_log_filename):
                if self.stopped():
                    break

                if self.skip_line(line=line):
                    continue

                cols = self.split_line(line=line)

                for metric in ['lat_mean', 'lat_med', 'lat_perc_95', 'lat_perc_99', 'lat_perc_999', 'lat_max']:
                    if metric_value := self.get_metric_value(columns=cols, metric_name=metric):
                        self.set_metric(metric, convert_metric_to_ms(metric_value))

                if ops := self.get_metric_value(columns=cols, metric_name='ops'):
                    self.set_metric('ops', float(ops))

                if errors := cols[self.metrics_positions.errors]:
                    self.set_metric('errors', int(errors))

class CassandraStressExporter(StressExporter):

    def __init__(self, instance_name: str, metrics: NemesisMetrics, stress_operation: str, stress_log_filename: str,
                 loader_idx: int, cpu_idx: int = 1):  # pylint: disable=too-many-arguments

        self.keyspace_regex = re.compile(r'.*Keyspace:\s(.*?)$')
        super(CassandraStressExporter, self).__init__(instance_name, metrics, stress_operation, stress_log_filename, loader_idx,
                                                      cpu_idx)

    def create_metrix_gauge(self):
        gauge_name = f'collectd_cassandra_stress_{self.stress_operation}_gauge'
        if gauge_name not in self.METRICS_GAUGES:
            self.METRICS_GAUGES[gauge_name] = self.metrics.create_gauge(
                gauge_name,
                'Gauge for cassandra stress metrics',
                [f'cassandra_stress_{self.stress_operation}', 'instance', 'loader_idx', 'cpu_idx', 'type', 'keyspace'])
        return gauge_name

    def merics_position_in_log(self) -> MetricsPosition:
        return MetricsPosition(ops=2, lat_mean=5, lat_med=6, lat_perc_95=7, lat_perc_99=8, lat_perc_999=9,
                               lat_max=10, errors=13)

    def skip_line(self, line: str) -> bool:
        if not self.keyspace:
            if 'Keyspace:' in line:
                self.keyspace = self.keyspace_regex.match(line).groups()[0]
        # If line starts with 'total,' - skip this line
        return not 'total,' in line

    @staticmethod
    def split_line(line: str) -> list:
        return [element.strip() for element in line.split(',')]

class ScyllaBenchStressExporter(StressExporter):

    def __init__(self, instance_name: str, metrics: NemesisMetrics, stress_operation: str, stress_log_filename: str,
                 loader_idx: int, cpu_idx: int = 1):  # pylint: disable=too-many-arguments

        super(ScyllaBenchStressExporter, self).__init__(instance_name, metrics, stress_operation, stress_log_filename,
                                                        loader_idx, cpu_idx)

    def create_metrix_gauge(self) -> str:
        gauge_name = f'collectd_scylla_bench_stress_{self.stress_operation}_gauge'
        if gauge_name not in self.METRICS_GAUGES:
            self.METRICS_GAUGES[gauge_name] = self.metrics.create_gauge(
                gauge_name,
                'Gauge for scylla-bench stress metrics',
                [f'scylla_bench_stress_{self.stress_operation}', 'instance', 'loader_idx', 'cpu_idx', 'type', 'keyspace'])
        return gauge_name

    def merics_position_in_log(self) -> MetricsPosition:
        # Enumerate stress metric position in the log. Example:
        # time                operations/s   rows/s   errors  max              99.9th           99th             95th             90th             median        mean
        # 1.033603151s                3439    34390        0  71.434239ms      70.713343ms      62.685183ms      2.818047ms       1.867775ms       1.048575ms    2.947276ms

        return MetricsPosition(ops=1, lat_mean=10, lat_med=9, lat_perc_95=7, lat_perc_99=6, lat_perc_999=5,
                               lat_max=4, errors=3)

    def skip_line(self, line) -> bool:
        # If line is not starts with numeric ended by "s" - skip this line.
        # Example:
        #    Client compression:  true
        #    1.004777157s                2891    28910        0  67.829759ms      64.290815ms      58.327039ms      4.653055ms       3.244031µs       1.376255µs

        line_splitted = (line or '').split()
        if not line_splitted or not line_splitted[0].endswith('s'):
            return True  # skip the line

        try:
            _ = float(line_splitted[0][:-1])
            return False  # The line hold the metrics, don't skip the line
        except ValueError:
            return True  # skip the line

    @staticmethod
    def split_line(line: str) -> list:
        return [element.strip() for element in line.split()]
<commit_msg>fix(loader): pylint issues<commit_after># This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2016 ScyllaDB

import os
import re
from abc import abstractmethod, ABCMeta
import time
import logging
from typing import NamedTuple

from sdcm.prometheus import NemesisMetrics
from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms

LOGGER = logging.getLogger(__name__)

class MetricsPosition(NamedTuple):
    ops: int
    lat_mean: int
    lat_med: int
    lat_perc_95: int
    lat_perc_99: int
    lat_perc_999: int
    lat_max: int
    errors: int

# pylint: disable=too-many-instance-attributes
class StressExporter(FileFollowerThread, metaclass=ABCMeta):
    METRICS_GAUGES = {}

    # pylint: disable=too-many-arguments
    def __init__(self, instance_name: str, metrics: NemesisMetrics, stress_operation: str, stress_log_filename: str,
                 loader_idx: int, cpu_idx: int = 1):
        super().__init__()
        self.metrics = metrics
        self.stress_operation = stress_operation
        self.stress_log_filename = stress_log_filename
        gauge_name = self.create_metrix_gauge()
        self.stress_metric = self.METRICS_GAUGES[gauge_name]
        self.instance_name = instance_name
        self.loader_idx = loader_idx
        self.cpu_idx = cpu_idx
        self.metrics_positions = self.merics_position_in_log()
        self.keyspace = ''

    @abstractmethod
    def merics_position_in_log(self) -> MetricsPosition:
        ...

    @abstractmethod
    def create_metrix_gauge(self) -> str:
        ...

    def set_metric(self, name: str, value: float) -> None:
        self.stress_metric.labels(0, self.instance_name, self.loader_idx, self.cpu_idx, name, self.keyspace).set(value)

    def clear_metrics(self) -> None:
        if self.stress_metric:
            for metric_name in self.metrics_positions._fields:
                self.set_metric(metric_name, 0.0)

    @staticmethod
    @abstractmethod
    def skip_line(line: str) -> bool:
        ...

    @staticmethod
    @abstractmethod
    def split_line(line: str) -> list:
        ...

    def get_metric_value(self, columns: list, metric_name: str) -> str:
        try:
            value = columns[getattr(self.metrics_positions, metric_name)]
        except (ValueError, IndexError) as exc:
            value = ''
            LOGGER.warning("Faile to get %s metric value. Error: %s", metric_name, str(exc))

        return value

    def run(self):
        while not self.stopped():
            exists = os.path.isfile(self.stress_log_filename)
            if not exists:
                time.sleep(0.5)
                continue

            for line in self.follow_file(self.stress_log_filename):
                if self.stopped():
                    break

                if self.skip_line(line=line):
                    continue

                cols = self.split_line(line=line)

                for metric in ['lat_mean', 'lat_med', 'lat_perc_95', 'lat_perc_99', 'lat_perc_999', 'lat_max']:
                    if metric_value := self.get_metric_value(columns=cols, metric_name=metric):
                        self.set_metric(metric, convert_metric_to_ms(metric_value))

                if ops := self.get_metric_value(columns=cols, metric_name='ops'):
                    self.set_metric('ops', float(ops))

                if errors := cols[self.metrics_positions.errors]:
                    self.set_metric('errors', int(errors))

class CassandraStressExporter(StressExporter):
    # pylint: disable=too-many-arguments
    def __init__(self, instance_name: str, metrics: NemesisMetrics, stress_operation: str, stress_log_filename: str,
                 loader_idx: int, cpu_idx: int = 1):

        self.keyspace_regex = re.compile(r'.*Keyspace:\s(.*?)$')
        super().__init__(instance_name, metrics, stress_operation, stress_log_filename, loader_idx,
                         cpu_idx)

    def create_metrix_gauge(self):
        gauge_name = f'collectd_cassandra_stress_{self.stress_operation}_gauge'
        if gauge_name not in self.METRICS_GAUGES:
            self.METRICS_GAUGES[gauge_name] = self.metrics.create_gauge(
                gauge_name,
                'Gauge for cassandra stress metrics',
                [f'cassandra_stress_{self.stress_operation}', 'instance', 'loader_idx', 'cpu_idx', 'type', 'keyspace'])
        return gauge_name

    def merics_position_in_log(self) -> MetricsPosition:
        return MetricsPosition(ops=2, lat_mean=5, lat_med=6, lat_perc_95=7, lat_perc_99=8, lat_perc_999=9,
                               lat_max=10, errors=13)

    def skip_line(self, line: str) -> bool:
        if not self.keyspace:
            if 'Keyspace:' in line:
                self.keyspace = self.keyspace_regex.match(line).groups()[0]
        # If line starts with 'total,' - skip this line
        return not 'total,' in line

    @staticmethod
    def split_line(line: str) -> list:
        return [element.strip() for element in line.split(',')]

class ScyllaBenchStressExporter(StressExporter):

    def create_metrix_gauge(self) -> str:
        gauge_name = f'collectd_scylla_bench_stress_{self.stress_operation}_gauge'
        if gauge_name not in self.METRICS_GAUGES:
            self.METRICS_GAUGES[gauge_name] = self.metrics.create_gauge(
                gauge_name,
                'Gauge for scylla-bench stress metrics',
                [f'scylla_bench_stress_{self.stress_operation}', 'instance', 'loader_idx', 'cpu_idx', 'type', 'keyspace'])
        return gauge_name

    def merics_position_in_log(self) -> MetricsPosition:
        # Enumerate stress metric position in the log. Example:
        # time  operations/s    rows/s   errors  max   99.9th   99th      95th     90th       median        mean
        # 1.033603151s    3439    34390    0  71.434239ms   70.713343ms   62.685183ms    2.818047ms  1.867775ms 1.048575ms  2.947276ms

        return MetricsPosition(ops=1, lat_mean=10, lat_med=9, lat_perc_95=7, lat_perc_99=6, lat_perc_999=5,
                               lat_max=4, errors=3)

    def skip_line(self, line) -> bool:
        # If line is not starts with numeric ended by "s" - skip this line.
        # Example:
        #    Client compression:  true
        #    1.004777157s       2891    28910     0  67.829759ms   64.290815ms    58.327039ms    4.653055ms   3.244031µs   1.376255µs

        line_splitted = (line or '').split()
        if not line_splitted or not line_splitted[0].endswith('s'):
            return True  # skip the line

        try:
            _ = float(line_splitted[0][:-1])
            return False  # The line hold the metrics, don't skip the line
        except ValueError:
            return True  # skip the line

    @staticmethod
    def split_line(line: str) -> list:
        return [element.strip() for element in line.split()]
<|endoftext|>


<commit_before>"""
Deploy this project in dev/stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander

"""

import os
import sys

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from commander.deploy import task, hostgroups
import commander_settings as settings

@task
def update_code(ctx, tag):
    """Update the code to a specific git reference (tag/sha/etc)."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('git fetch')
        ctx.local('git checkout -f %s' % tag)
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')

@task
def update_info(ctx):
    """Write info about the current state to a publicly visible file."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('date')
        ctx.local('git branch')
        ctx.local('git log -3')
        ctx.local('git status')
        ctx.local('git submodule status')
        with ctx.lcd('locale'):
            ctx.local('svn info')
            ctx.local('svn status')

        ctx.local('git rev-parse HEAD > static/revision')

@task
def clean(ctx):
    """Clean .gitignore and .pyc files."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local("find . -type f -name '.gitignore' -or -name '*.pyc' -delete")

@task
def update_assets(ctx):
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py collectstatic --noinput')
        # LANG=en_US.UTF-8 is sometimes necessary for the YUICompressor.
        ctx.local('LANG=en_US.UTF8 python2.6 manage.py compress_assets')

@task
def update_db(ctx):
    """
    Update the database schema, if necessary.

    Uses schematic by default. Change to south if you need to.
    """
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py syncdb --noinput')
        ctx.local('python2.6 manage.py migrate --noinput')
        ctx.local('python2.6 manage.py migrate --list')

@task
def update_locales(ctx):
    """
    Update a locale directory from SVN.

    Assumes localizations 1) exist, 2) are in SVN, 3) are in SRC_DIR/locale and
    4) have a compile-mo.sh script. This should all be pretty standard, but
    change it if you need to.
    """
    with ctx.lcd(os.path.join(settings.SRC_DIR, 'locale')):
        ctx.local('svn up')
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('./bin/compile-mo.sh locale/')

@task
def update_product_details(ctx):
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py update_product_details')

@task
def checkin_changes(ctx):
    """Use the local, IT-written deploy script to check in changes."""
    ctx.local(settings.DEPLOY_SCRIPT)

@hostgroups(settings.WEB_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
def deploy_app(ctx):
    """Call the remote update script to push changes to webheads."""
    ctx.remote('touch %s' % settings.REMOTE_WSGI)

@hostgroups(settings.CELERY_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
def update_celery(ctx):
    """Update and restart Celery."""
    ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
    ctx.remote('service %s restart' % settings.CELERY_SERVICE)

@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    """Update code to pick up changes to this file."""
    update_code(ref)
    update_info()
    clean()

@task
def update(ctx):
    update_assets()
    update_locales()
    update_db()
    update_product_details()

@task
def deploy(ctx):
    checkin_changes()
    deploy_app()
    update_celery()

@task
def update_site(ctx, tag):
    """Update the app to prep for deployment."""
    pre_update(tag)
    update()
<commit_msg>Update deploy script as per bug 843216.<commit_after>"""
Deploy this project in dev/stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander

"""

import os
import sys

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from commander.deploy import task, hostgroups
import commander_settings as settings

@task
def update_code(ctx, tag):
    """Update the code to a specific git reference (tag/sha/etc)."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('git fetch')
        ctx.local('git checkout -f %s' % tag)
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')

@task
def update_info(ctx):
    """Write info about the current state to a publicly visible file."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('date')
        ctx.local('git branch')
        ctx.local('git log -3')
        ctx.local('git status')
        ctx.local('git submodule status')
        with ctx.lcd('locale'):
            ctx.local('svn info')
            ctx.local('svn status')

        ctx.local('git rev-parse HEAD > static/revision')

@task
def clean(ctx):
    """Clean .gitignore and .pyc files."""
    with ctx.lcd(settings.SRC_DIR):
        ctx.local("find . -type f -name '.gitignore' -or -name '*.pyc' -delete")

@task
def update_assets(ctx):
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py collectstatic --noinput')
        # LANG=en_US.UTF-8 is sometimes necessary for the YUICompressor.
        ctx.local('LANG=en_US.UTF8 python2.6 manage.py compress_assets')

@task
def update_db(ctx):
    """
    Update the database schema, if necessary.

    Uses schematic by default. Change to south if you need to.
    """
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py syncdb --noinput')
        ctx.local('python2.6 manage.py migrate --noinput')
        ctx.local('python2.6 manage.py migrate --list')

@task
def update_locales(ctx):
    """
    Update a locale directory from SVN.

    Assumes localizations 1) exist, 2) are in SVN, 3) are in SRC_DIR/locale and
    4) have a compile-mo.sh script. This should all be pretty standard, but
    change it if you need to.
    """
    with ctx.lcd(os.path.join(settings.SRC_DIR, 'locale')):
        ctx.local('svn up')
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('./bin/compile-mo.sh locale/')

@task
def update_product_details(ctx):
    with ctx.lcd(settings.SRC_DIR):
        ctx.local('python2.6 manage.py update_product_details')

@task
def checkin_changes(ctx):
    """Use the local, IT-written deploy script to check in changes."""
    ctx.local(settings.DEPLOY_SCRIPT)

@hostgroups(settings.WEB_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
def deploy_app(ctx):
    """Call the remote update script to push changes to webheads."""
    ctx.remote('touch %s' % settings.REMOTE_WSGI)

@hostgroups(settings.CELERY_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
def update_celery(ctx):
    """Update and restart Celery."""
    ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
    ctx.remote('supervisorctl restart %s' % settings.CELERY_SERVICE)

@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    """Update code to pick up changes to this file."""
    update_code(ref)
    update_info()
    clean()

@task
def update(ctx):
    update_assets()
    update_locales()
    update_db()
    update_product_details()

@task
def deploy(ctx):
    checkin_changes()
    deploy_app()
    update_celery()

@task
def update_site(ctx, tag):
    """Update the app to prep for deployment."""
    pre_update(tag)
    update()
<|endoftext|>