<commit_before>"""Bash-style brace expansion"""
import re
import string
from itertools import chain, product

__version__ = '0.2'

__all__ = ['braceexpand', 'alphabet', 'UnbalancedBracesError']

class UnbalancedBracesError(ValueError): pass

alphabet = string.uppercase + string.lowercase

int_range_re = re.compile(r'^(\d+)\.\.(\d+)(?:\.\.-?(\d+))?$')
char_range_re = re.compile(r'^([A-Za-z])\.\.([A-Za-z])(?:\.\.-?(\d+))?$')

def braceexpand(pattern, escape=True):
    """braceexpand(pattern) -> iterator over generated strings

    Returns an iterator over the strings resulting from brace expansion
    of pattern. This function implements Brace Expansion as described in
    bash(1), with the following limitations:

    * A pattern containing unbalanced braces will raise an
      UnbalancedBracesError exception. In bash, unbalanced braces will either
      be partly expanded or ignored.

    * A mixed-case character range like '{Z..a}' or '{a..Z}' will not
      include the characters '[]^_`' between 'Z' and 'a'.

    When escape is True (the default), characters in pattern can be
    prefixed with a backslash to cause them not to be interpreted as
    special characters for brace expansion (such as '{', '}', ',').
    To pass through a a literal backslash, double it ('\\\\').

    When escape is False, backslashes in pattern have no special
    meaning and will be preserved in the output.

    Examples:

    >>> from braceexpand import braceexpand

    # Integer range
    >>> list(braceexpand('item{1..3}'))
    ['item1', 'item2', 'item3']

    # Character range
    >>> list(braceexpand('{a..c}'))
    ['a', 'b', 'c']

    # Sequence
    >>> list(braceexpand('index.html{,.backup}'))
    ['index.html', 'index.html.backup']

    # Nested patterns
    >>> list(braceexpand('python{2.{5..7},3.{2,3}}'))
    ['python2.5', 'python2.6', 'python2.7', 'python3.2', 'python3.3']

    # Prefixing an integer with zero causes all numbers to be padded to
    # the same width.
    >>> list(braceexpand('{07..10}'))
    ['07', '08', '09', '10']

    # An optional increment can be specified for ranges.
    >>> list(braceexpand('{a..g..2}'))
    ['a', 'c', 'e', 'g']

    # Ranges can go in both directions.
    >>> list(braceexpand('{4..1}'))
    ['4', '3', '2', '1']

    # Unbalanced braces raise an exception.
    >>> list(braceexpand('{1{2,3}'))
    Traceback (most recent call last):
        ...
    UnbalancedBracesError: Unbalanced braces: '{1{2,3}'

    # By default, the backslash is the escape character.
    >>> list(braceexpand(r'{1\{2,3}'))
    ['1{2', '3']

    # Setting 'escape' to False disables backslash escaping.
    >>> list(braceexpand(r'\{1,2}', escape=False))
    ['\\\\1', '\\\\2']

    """
    return (_flatten(t, escape) for t in parse_pattern(pattern, escape))

def parse_pattern(pattern, escape):
    # pattern -> product(*parts)
    start = 0
    pos = 0
    bracketdepth = 0
    items = []

    #print 'pattern:', pattern
    while pos < len(pattern):
        if escape and pattern[pos] == '\\':
            pos += 2
            continue
        elif pattern[pos] == '{':
            if bracketdepth == 0 and pos > start:
                #print 'literal:', pattern[start:pos]
                items.append([pattern[start:pos]])
                start = pos
            bracketdepth += 1
        elif pattern[pos] == '}':
            bracketdepth -= 1
            if bracketdepth == 0:
                #print 'expression:', pattern[start+1:pos]
                items.append(parse_expression(pattern[start+1:pos], escape))
                start = pos + 1 # skip the closing brace
        pos += 1

    if bracketdepth != 0: # unbalanced braces
        raise UnbalancedBracesError("Unbalanced braces: '%s'" % pattern)

    if start < pos:
        #print 'literal:', pattern[start:]
        items.append([pattern[start:]])

    return product(*items)

def parse_expression(expr, escape):
    int_range_match = int_range_re.match(expr)
    if int_range_match:
        return make_int_range(*int_range_match.groups())

    char_range_match = char_range_re.match(expr)
    if char_range_match:
        return make_char_range(*char_range_match.groups())

    return parse_sequence(expr, escape)

def parse_sequence(seq, escape):
    # sequence -> chain(*sequence_items)
    start = 0
    pos = 0
    bracketdepth = 0
    items = []

    #print 'sequence:', seq
    while pos < len(seq):
        if escape and seq[pos] == '\\':
            pos += 2
            continue
        elif seq[pos] == '{':
            bracketdepth += 1
        elif seq[pos] == '}':
            bracketdepth -= 1
        elif seq[pos] == ',' and bracketdepth == 0:
            items.append(parse_pattern(seq[start:pos], escape))
            start = pos + 1 # skip the comma
        pos += 1

    if bracketdepth != 0 or not items: # unbalanced braces or not a sequence
        return iter(['{' + seq + '}'])

    # part after the last comma (may be the empty string)
    items.append(parse_pattern(seq[start:], escape))
    return chain(*items)

def make_int_range(start, end, step=None):
    padding = max([len(start), len(end)]) if '0' in (start[0], end[0]) else 0
    step = int(step) if step else 1
    start = int(start)
    end = int(end)
    r = xrange(start, end+1, step) if start < end else \
        xrange(start, end-1, -step)
    return (str(i).rjust(padding, '0') for i in r)

def make_char_range(start, end, step=None):
    step = int(step) if step else 1
    start = alphabet.index(start)
    end = alphabet.index(end)
    return alphabet[start:end+1:step] if start < end else \
           alphabet[start:end-1:-step]

escape_re = re.compile(r'\\(.)')

def _flatten(t, escape):
    l = []
    for item in t:
        if isinstance(item, tuple): l.extend(_flatten(item, escape))
        else: l.append(item)
    s = ''.join(l)
    # Strip escape characters from generated strings after expansion.
    return escape_re.sub(r'\1', s) if escape else s

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<commit_msg>Reset version to 0.1 before public release<commit_after>"""Bash-style brace expansion"""
import re
import string
from itertools import chain, product

__version__ = '0.1'

__all__ = ['braceexpand', 'alphabet', 'UnbalancedBracesError']

class UnbalancedBracesError(ValueError): pass

alphabet = string.uppercase + string.lowercase

int_range_re = re.compile(r'^(\d+)\.\.(\d+)(?:\.\.-?(\d+))?$')
char_range_re = re.compile(r'^([A-Za-z])\.\.([A-Za-z])(?:\.\.-?(\d+))?$')

def braceexpand(pattern, escape=True):
    """braceexpand(pattern) -> iterator over generated strings

    Returns an iterator over the strings resulting from brace expansion
    of pattern. This function implements Brace Expansion as described in
    bash(1), with the following limitations:

    * A pattern containing unbalanced braces will raise an
      UnbalancedBracesError exception. In bash, unbalanced braces will either
      be partly expanded or ignored.

    * A mixed-case character range like '{Z..a}' or '{a..Z}' will not
      include the characters '[]^_`' between 'Z' and 'a'.

    When escape is True (the default), characters in pattern can be
    prefixed with a backslash to cause them not to be interpreted as
    special characters for brace expansion (such as '{', '}', ',').
    To pass through a a literal backslash, double it ('\\\\').

    When escape is False, backslashes in pattern have no special
    meaning and will be preserved in the output.

    Examples:

    >>> from braceexpand import braceexpand

    # Integer range
    >>> list(braceexpand('item{1..3}'))
    ['item1', 'item2', 'item3']

    # Character range
    >>> list(braceexpand('{a..c}'))
    ['a', 'b', 'c']

    # Sequence
    >>> list(braceexpand('index.html{,.backup}'))
    ['index.html', 'index.html.backup']

    # Nested patterns
    >>> list(braceexpand('python{2.{5..7},3.{2,3}}'))
    ['python2.5', 'python2.6', 'python2.7', 'python3.2', 'python3.3']

    # Prefixing an integer with zero causes all numbers to be padded to
    # the same width.
    >>> list(braceexpand('{07..10}'))
    ['07', '08', '09', '10']

    # An optional increment can be specified for ranges.
    >>> list(braceexpand('{a..g..2}'))
    ['a', 'c', 'e', 'g']

    # Ranges can go in both directions.
    >>> list(braceexpand('{4..1}'))
    ['4', '3', '2', '1']

    # Unbalanced braces raise an exception.
    >>> list(braceexpand('{1{2,3}'))
    Traceback (most recent call last):
        ...
    UnbalancedBracesError: Unbalanced braces: '{1{2,3}'

    # By default, the backslash is the escape character.
    >>> list(braceexpand(r'{1\{2,3}'))
    ['1{2', '3']

    # Setting 'escape' to False disables backslash escaping.
    >>> list(braceexpand(r'\{1,2}', escape=False))
    ['\\\\1', '\\\\2']

    """
    return (_flatten(t, escape) for t in parse_pattern(pattern, escape))

def parse_pattern(pattern, escape):
    # pattern -> product(*parts)
    start = 0
    pos = 0
    bracketdepth = 0
    items = []

    #print 'pattern:', pattern
    while pos < len(pattern):
        if escape and pattern[pos] == '\\':
            pos += 2
            continue
        elif pattern[pos] == '{':
            if bracketdepth == 0 and pos > start:
                #print 'literal:', pattern[start:pos]
                items.append([pattern[start:pos]])
                start = pos
            bracketdepth += 1
        elif pattern[pos] == '}':
            bracketdepth -= 1
            if bracketdepth == 0:
                #print 'expression:', pattern[start+1:pos]
                items.append(parse_expression(pattern[start+1:pos], escape))
                start = pos + 1 # skip the closing brace
        pos += 1

    if bracketdepth != 0: # unbalanced braces
        raise UnbalancedBracesError("Unbalanced braces: '%s'" % pattern)

    if start < pos:
        #print 'literal:', pattern[start:]
        items.append([pattern[start:]])

    return product(*items)

def parse_expression(expr, escape):
    int_range_match = int_range_re.match(expr)
    if int_range_match:
        return make_int_range(*int_range_match.groups())

    char_range_match = char_range_re.match(expr)
    if char_range_match:
        return make_char_range(*char_range_match.groups())

    return parse_sequence(expr, escape)

def parse_sequence(seq, escape):
    # sequence -> chain(*sequence_items)
    start = 0
    pos = 0
    bracketdepth = 0
    items = []

    #print 'sequence:', seq
    while pos < len(seq):
        if escape and seq[pos] == '\\':
            pos += 2
            continue
        elif seq[pos] == '{':
            bracketdepth += 1
        elif seq[pos] == '}':
            bracketdepth -= 1
        elif seq[pos] == ',' and bracketdepth == 0:
            items.append(parse_pattern(seq[start:pos], escape))
            start = pos + 1 # skip the comma
        pos += 1

    if bracketdepth != 0 or not items: # unbalanced braces or not a sequence
        return iter(['{' + seq + '}'])

    # part after the last comma (may be the empty string)
    items.append(parse_pattern(seq[start:], escape))
    return chain(*items)

def make_int_range(start, end, step=None):
    padding = max([len(start), len(end)]) if '0' in (start[0], end[0]) else 0
    step = int(step) if step else 1
    start = int(start)
    end = int(end)
    r = xrange(start, end+1, step) if start < end else \
        xrange(start, end-1, -step)
    return (str(i).rjust(padding, '0') for i in r)

def make_char_range(start, end, step=None):
    step = int(step) if step else 1
    start = alphabet.index(start)
    end = alphabet.index(end)
    return alphabet[start:end+1:step] if start < end else \
           alphabet[start:end-1:-step]

escape_re = re.compile(r'\\(.)')

def _flatten(t, escape):
    l = []
    for item in t:
        if isinstance(item, tuple): l.extend(_flatten(item, escape))
        else: l.append(item)
    s = ''.join(l)
    # Strip escape characters from generated strings after expansion.
    return escape_re.sub(r'\1', s) if escape else s

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division

import logging

import numpy as np

import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

from model import Model, default_weights
from utils.unrolled_scan import unrolled_scan

_logger = logging.getLogger(__name__)

theano.config.exception_verbosity = 'high'
theano_rng = RandomStreams(seed=2341)

def f_replicate_batch(X, repeat):
    X_ = X.dimshuffle((0, 'x', 1))
    X_ = X_ + T.zeros((X.shape[0], repeat, X.shape[1]), dtype='float32')
    X_ = X_.reshape( [X_.shape[0]*repeat, X.shape[1]] )
    return X_

class ISB(Model):
    def __init__(self, **hyper_params):
        super(ISB, self).__init__()

        self.register_hyper_param('n_vis', help='no. observed binary variables')
        self.register_hyper_param('n_hid', help='no. latent binary variables')
        self.register_hyper_param('n_qhid', help='no. latent binary variables')
        self.register_hyper_param('clamp_sigmoid', default=False)
        self.register_hyper_param('n_samples', default=100)
        self.register_hyper_param('unroll_scan', default=1)

        # Sigmoid Belief Layer
        self.register_model_param('P_a', help='P hidden prior', default=lambda: np.ones(self.n_hid)/2.)
        self.register_model_param('P_b', help='P visible bias', default=lambda: np.zeros(self.n_vis))
        self.register_model_param('P_W', help='P weights', default=lambda: default_weights(self.n_hid, self.n_vis) )

        # Conditional NADE
        self.register_model_param('Q_b',  help='Q visible bias', default=lambda: np.zeros(self.n_hid))
        self.register_model_param('Q_c',  help='Q hidden bias' , default=lambda: np.zeros(self.n_qhid))
        self.register_model_param('Q_Ub', help='Q cond. weights Ub', default=lambda: default_weights(self.n_vis, self.n_hid) )
        self.register_model_param('Q_Uc', help='Q cond. weights Uc', default=lambda: default_weights(self.n_vis, self.n_qhid) )
        self.register_model_param('Q_W',  help='Q encoder weights', default=lambda: default_weights(self.n_hid, self.n_qhid) )
        self.register_model_param('Q_V',  help='Q decoder weights', default=lambda: default_weights(self.n_qhid, self.n_hid) )

        self.set_hyper_params(hyper_params)

    def f_sigmoid(self, x):
        if self.clamp_sigmoid:
            return T.nnet.sigmoid(x)*0.9999 + 0.000005
        else:
            return T.nnet.sigmoid(x)

    def f_q(self, X, Y):
        n_vis, n_qhid, n_cond = self.get_hyper_params(['n_vis', 'n_qhid', 'n_cond'])
        b, c, W, V, Ub, Uc = self.get_model_params(['b', 'c', 'W', 'V', 'Ub', 'Uc'])

        batch_size = X.shape[0]

        vis = X
        cond = Y

        learning_rate = T.fscalar('learning_rate')
        learning_rate.tag.test_value = 0.0

        #------------------------------------------------------------------
        b_cond = b + T.dot(cond, Ub)    # shape (batch, n_vis)
        c_cond = c + T.dot(cond, Uc)    # shape (batch, n_hid)

        a_init    = c_cond
        post_init = T.zeros([batch_size], dtype=np.float32)

        def one_iter(vis_i, Wi, Vi, bi, a, post):
            hid  = self.f_sigmoid(a)
            pi   = self.f_sigmoid(T.dot(hid, Vi) + bi)
            post = post + T.cast(T.log(pi*vis_i + (1-pi)*(1-vis_i)), dtype='float32')
            a    = a + T.outer(vis_i, Wi)
            return a, post

        [a, post], updates = unrolled_scan(
                    fn=one_iter,
                    sequences=[vis.T, W, V.T, b_cond.T],
                    outputs_info=[a_init, post_init],
                    unroll=self.unroll_scan
                )
        return post[-1,:]

    def f_loglikelihood(self, X, Y=None):
        n_samples, = self.get_hyper_params(['n_samples'])
        #b, c, W, V, Ub, Uc = self.get_model_params(['b', 'c', 'W', 'V', 'Ub', 'Uc'])

        batch_size = X.shape[0]

        # grow X by factor of samples
        X = f_replicate_batch(X, n_samples)

        # Get samples from q
        H, lQ = self.f_q_sample(X)   # batch*n_samples

        # Calculate log P(X, H)
        lP = self.f_p(X, H)

        # Approximate log P(X)
        lPx = T.shape_padright(lP)
        lPx = lPx.reshape( (batch_size, n_samples) )
        lPx_max = T.max(lPx, axis=1)
        lPx = lPx - T.shape_padright(lPx_max)
        lPx = T.log(T.sum(T.exp(lPx), axis=1))
        lPx = T.shape_padright(lPx)
        #lPx = f_replicate_batch(lPx, n_samples)

        lP = lP.reshape( (batch_size, n_samples) )
        lQ = lQ.reshape( (batch_size, n_samples) )

        # calc. sampling weights
        w = T.exp(lP-lQ-lPx)

        print "lP:  ", lP.tag.test_value.shape
        print "lQ:  ", lQ.tag.test_value.shape
        print "lPx: ", lPx.tag.test_value.shape
        print "w:   ", w.tag.test_value.shape

        return lP, lQ, H, w

    #------------------------ P ---------------------------------------------
    def f_p(self, X, H):
        W, a, b = self.get_model_params(['P_W', 'P_a', 'P_b'])

        # Prior P(H)
        lpH = T.cast(T.log(a*H + (1-a)*(1-H)), dtype='float32')
        lpH = T.sum(lpH, axis=1)

        # Posterior P(X|H)
        pX = self.f_sigmoid(T.dot(H, W) + b)
        lpXH = T.cast(T.log(pX*X + (1-pX)*(1-X)), dtype='float32')
        lpXH = T.sum(lpXH, axis=1)

        return lpH + lpXH

    #------------------------ Q ---------------------------------------------
    def f_q_sample(self, X):
        n_vis, n_hid, n_qhid = self.get_hyper_params(['n_vis', 'n_hid', 'n_qhid'])
        b, c, W, V, Ub, Uc = self.get_model_params(['Q_b', 'Q_c', 'Q_W', 'Q_V', 'Q_Ub', 'Q_Uc'])

        cond = X
        batch_size = cond.shape[0]

        #------------------------------------------------------------------
        b_cond = b + T.dot(cond, Ub)    # shape (batch, n_vis)
        c_cond = c + T.dot(cond, Uc)    # shape (batch, n_hid)

        a_init    = c_cond
        post_init = T.zeros([batch_size], dtype=np.float32)
        vis_init  = T.zeros([batch_size], dtype=np.float32)

        def one_iter(Wi, Vi, bi, a, vis_i, post):
            hid  = self.f_sigmoid(a)
            pi   = self.f_sigmoid(T.dot(hid, Vi) + bi)
            vis_i = 1.*(theano_rng.uniform([batch_size]) <= pi)
            post  = post + T.cast(T.log(pi*vis_i + (1-pi)*(1-vis_i)), dtype='float32')
            a     = a + T.outer(vis_i, Wi)
            return a, vis_i, post

        [a, vis, post], updates = unrolled_scan(
                    fn=one_iter,
                    sequences=[W, V.T, b_cond.T],
                    outputs_info=[a_init, vis_init, post_init],
                    unroll=self.unroll_scan
                )

        return vis.T, post[-1,:]

<commit_msg>Progress on ISB<commit_after>#!/usr/bin/env python

from __future__ import division

import logging

import numpy as np

import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from theano.printing import Print

from model import Model, default_weights
from utils.unrolled_scan import unrolled_scan

_logger = logging.getLogger(__name__)

theano.config.exception_verbosity = 'high'
theano_rng = RandomStreams(seed=2341)

def f_replicate_batch(X, repeat):
    X_ = X.dimshuffle((0, 'x', 1))
    X_ = X_ + T.zeros((X.shape[0], repeat, X.shape[1]), dtype='float32')
    X_ = X_.reshape( [X_.shape[0]*repeat, X.shape[1]] )
    return X_

class ISB(Model):
    def __init__(self, **hyper_params):
        super(ISB, self).__init__()

        self.register_hyper_param('n_vis', help='no. observed binary variables')
        self.register_hyper_param('n_hid', help='no. latent binary variables')
        self.register_hyper_param('n_qhid', help='no. latent binary variables')
        self.register_hyper_param('clamp_sigmoid', default=True)
        self.register_hyper_param('n_samples', default=100)
        self.register_hyper_param('unroll_scan', default=1)

        # Sigmoid Belief Layer
        self.register_model_param('P_a', help='P hidden prior', default=lambda: np.zeros(self.n_hid))
        self.register_model_param('P_b', help='P visible bias', default=lambda: np.zeros(self.n_vis))
        self.register_model_param('P_W', help='P weights', default=lambda: default_weights(self.n_hid, self.n_vis) )

        # Conditional NADE
        self.register_model_param('Q_b',  help='Q visible bias', default=lambda: np.zeros(self.n_hid))
        self.register_model_param('Q_c',  help='Q hidden bias' , default=lambda: np.zeros(self.n_qhid))
        self.register_model_param('Q_Ub', help='Q cond. weights Ub', default=lambda: default_weights(self.n_vis, self.n_hid) )
        self.register_model_param('Q_Uc', help='Q cond. weights Uc', default=lambda: default_weights(self.n_vis, self.n_qhid) )
        self.register_model_param('Q_W',  help='Q encoder weights', default=lambda: default_weights(self.n_hid, self.n_qhid) )
        self.register_model_param('Q_V',  help='Q decoder weights', default=lambda: default_weights(self.n_qhid, self.n_hid) )

        self.set_hyper_params(hyper_params)
        self.Q_params = ['Q_b', 'Q_c', 'Q_Ub', 'Q_Uc', 'Q_W', 'Q_V']
        self.P_params = ['P_a', 'P_b', 'P_W']

    def f_sigmoid(self, x):
        if self.clamp_sigmoid:
            return T.nnet.sigmoid(x)*0.9999 + 0.000005
        else:
            return T.nnet.sigmoid(x)

    def f_loglikelihood(self, X, Y=None):
        n_hid, n_samples = self.get_hyper_params(['n_hid', 'n_samples'])
        #b, c, W, V, Ub, Uc = self.get_model_params(['b', 'c', 'W', 'V', 'Ub', 'Uc'])

        batch_size = X.shape[0]

        # grow X by factor of samples
        X = f_replicate_batch(X, n_samples)

        # Get samples from q
        #H, lQ = self.f_q_sample_flat(X)   # batch*n_samples
        H, lQ = self.f_q_sample(X)

        #H  = Print('H')(H)
        #lQ = Print('lQ')(lQ)

        # Calculate log P(X, H)
        lP = self.f_p(X, H)

        def logsumexp1(A):
            A = A.reshape( (batch_size, n_samples) )
            A_max = T.max(A, axis=1)
            A_ = A - T.shape_padright(A_max)
            A = T.log(T.sum(T.exp(A_), axis=1))
            A = A + A_max
            return T.shape_padright(A)

        # Approximate log P(X)
        lPx = logsumexp1(lP-lQ)

        #lP  = Print('lP')(lP)
        #lPx = Print('lPx')(lPx)

        H   =  H.reshape( (batch_size, n_samples, n_hid) )
        lP  = lP.reshape( (batch_size, n_samples) )
        lQ  = lQ.reshape( (batch_size, n_samples) )
        #lQl = lQl.reshape( (batch_size, n_samples) )

        # calc. sampling weights
        w = T.exp(lP-lQ-lPx)
        #w = Print('w')(w)

        #print "lP:  ", lP.tag.test_value.shape
        #print "lQ:  ", lQ.tag.test_value.shape
        #print "H:   ", w.tag.test_value.shape
        #print "lPx: ", lPx.tag.test_value.shape
        #print "w:   ", w.tag.test_value.shape

        return lP, lQ, H, w

    #------------------------ P ---------------------------------------------
    def f_p(self, X, H):
        W, a, b = self.get_model_params(['P_W', 'P_a', 'P_b'])

        # Prior P(H)
        p_hid = self.f_sigmoid(a)
        lpH = T.log(p_hid*H + (1-p_hid)*(1-H))
        lpH = T.sum(lpH, axis=1)

        # Posterior P(X|H)
        pX = self.f_sigmoid(T.dot(H, W) + b)
        lpXH = T.log(pX*X + (1-pX)*(1-X))
        lpXH = T.sum(lpXH, axis=1)

        lP = lpH + lpXH
        #lP  = Print('lP')(lP)
        return lP

    def f_p_sample(self, n_samples):
        n_vis, n_hid = self.get_hyper_params(['n_vis', 'n_hid'])
        W, a, b = self.get_model_params(['P_W', 'P_a', 'P_b'])

        # samples hiddens
        p_hid = self.f_sigmoid(a)
        H = T.cast(theano_rng.uniform((n_samples, n_hid)) <= p_hid, dtype='float32')

        # sample visible given hiddens
        p_vis = self.f_sigmoid(T.dot(H, W) + b)
        X = T.cast(theano_rng.uniform((n_samples, n_vis)) <= p_vis, dtype='float32')

        return H, X

    #------------------------ Q ---------------------------------------------
    def f_q_sample_flat(self, X):
        n_vis, n_hid, n_qhid = self.get_hyper_params(['n_vis', 'n_hid', 'n_qhid'])
        b, c, W, V, Ub, Uc = self.get_model_params(['Q_b', 'Q_c', 'Q_W', 'Q_V', 'Q_Ub', 'Q_Uc'])

        H_ = np.zeros((256, 8), dtype='float32')
        for i in xrange(256):
            for j in xrange(8):
                if i & (1 << j):
                    H_[i,7-j] = 1.

        batch_size = 1 # X.shape[0] // 256

        #H_ = f_replicate_batch(theano.shared(H_), batch_size)
        H_ = theano.shared(H_)
        Q_ = theano.shared((np.ones(256*batch_size)/256))
        return H_, Q_

    def f_q_sample(self, X):
        n_vis, n_hid, n_qhid = self.get_hyper_params(['n_vis', 'n_hid', 'n_qhid'])
        b, c, W, V, Ub, Uc = self.get_model_params(['Q_b', 'Q_c', 'Q_W', 'Q_V', 'Q_Ub', 'Q_Uc'])

        cond = X
        batch_size = cond.shape[0]

        #------------------------------------------------------------------
        b_cond = b + T.dot(cond, Ub)    # shape (batch, n_vis)
        c_cond = c + T.dot(cond, Uc)    # shape (batch, n_hid)

        a_init    = c_cond
        post_init = T.zeros([batch_size], dtype=np.float32)
        vis_init  = T.zeros([batch_size], dtype=np.float32)

        def one_iter(Wi, Vi, bi, a, vis_i, post):
            hid  = self.f_sigmoid(a)
            pi   = self.f_sigmoid(T.dot(hid, Vi) + bi)
            vis_i = 1.*(theano_rng.uniform([batch_size]) <= pi)
            post  = post + vis_i*T.log(pi) + (1-vis_i)*T.log(1-pi)
            a     = a + T.outer(vis_i, Wi)
            return a, vis_i, post

        [a, vis, post], updates = unrolled_scan(
                    fn=one_iter,
                    sequences=[W, V.T, b_cond.T],
                    outputs_info=[a_init, vis_init, post_init],
                    unroll=self.unroll_scan
                )

        return vis.T, post[-1,:]

    def f_q(self, vis, cond):
        n_vis, n_hid, n_qhid = self.get_hyper_params(['n_vis', 'n_hid', 'n_qhid'])
        b, c, W, V, Ub, Uc = self.get_model_params(['Q_b', 'Q_c', 'Q_W', 'Q_V', 'Q_Ub', 'Q_Uc'])

        batch_size = cond.shape[0]

        #------------------------------------------------------------------
        b_cond = b + T.dot(cond, Ub)    # shape (batch, n_vis)
        c_cond = c + T.dot(cond, Uc)    # shape (batch, n_hid)

        a_init    = c_cond
        post_init = T.zeros([batch_size], dtype=np.float32)

        def one_iter(vis_i, Wi, Vi, bi, a, post):
            hid  = self.f_sigmoid(a)
            pi   = self.f_sigmoid(T.dot(hid, Vi) + bi)
            #post = post + T.cast(T.log(pi*vis_i + (1-pi)*(1-vis_i)), dtype='float32')
            post = post + vis_i*T.log(pi) + (1-vis_i)*T.log(1-pi)
            a    = a + T.outer(vis_i, Wi)
            return a, post

        [a, post], updates = unrolled_scan(
                    fn=one_iter,
                    sequences=[vis.T, W, V.T, b_cond.T],
                    outputs_info=[a_init, post_init],
                    unroll=self.unroll_scan
                )
        return post[-1,:]

<|endoftext|>


<commit_before>import drake
import markdown2

class Renderer(drake.Builder):

  def __init__(self, source, target = None):
    self.__source = source
    self.__target = target or drake.Node(
      self.__source.path().with_extension('html')
    )
    drake.Builder.__init__(self, [self.__source], [self.__target])

  def execute(self):
    self.output('Render %s' % self.__target)
    with open(str(self.__source.path())) as input:
      with open(str(self.__target.path()), 'w') as output:
        output.write(
          markdown2.markdown(input.read(),
                             extras = ['header-ids'])
        )
    return True

class Source(drake.Node):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    Renderer(self)

drake.Node.extensions['md'] = Source
<commit_msg>Revert "markdown: Use markdown2 instead of mistune."<commit_after>import drake
import mistune

class Renderer(drake.Builder):

  def __init__(self, source, target = None):
    self.__source = source
    self.__target = target or drake.Node(
      self.__source.path().with_extension('html')
    )
    drake.Builder.__init__(self, [self.__source], [self.__target])

  def execute(self):
    self.output('Render %s' % self.__target)
    renderer = mistune.Renderer()
    markdown = mistune.Markdown(
      renderer = renderer,
      parse_inline_html = True,
    )
    with open(str(self.__source.path())) as input:
      with open(str(self.__target.path()), 'w') as output:
        output.write(markdown(input.read()))
    return True

class Source(drake.Node):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    Renderer(self)

drake.Node.extensions['md'] = Source
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from distutils.dir_util import copy_tree

class Savanna(MakefilePackage):
    """CODARcode Savanna runtime framework for high performance,
    workflow management using Swift/T and ADIOS.
    """

    homepage = "https://github.com/CODARcode/savanna"
    url = "https://github.com/CODARcode/savanna/archive/v0.5.tar.gz"

    version('develop', git='https://github.com/CODARcode/savanna.git',
            branch='master', submodules=True)
    version('0.5', git='https://github.com/CODARcode/savanna.git',
            tag='0.5', submodules=True)

    variant('tau', default=False, description='Enable TAU profiling support')

    depends_on('mpi')
    depends_on('stc')
    depends_on('adios +fortran +staging')
    depends_on('mpix-launch-swift')
    depends_on('tau', when='+tau')

    def install(self, spec, prefix):
        copy_tree('.', prefix)
<commit_msg>savanna depends on adios+sz (#4954)<commit_after>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from distutils.dir_util import copy_tree

class Savanna(MakefilePackage):
    """CODARcode Savanna runtime framework for high performance,
    workflow management using Swift/T and ADIOS.
    """

    homepage = "https://github.com/CODARcode/savanna"
    url = "https://github.com/CODARcode/savanna/archive/v0.5.tar.gz"

    version('develop', git='https://github.com/CODARcode/savanna.git',
            branch='master', submodules=True)
    version('0.5', git='https://github.com/CODARcode/savanna.git',
            tag='0.5', submodules=True)

    variant('tau', default=False, description='Enable TAU profiling support')

    depends_on('mpi')
    depends_on('stc')
    depends_on('adios +fortran +zlib +sz +zfp +staging')
    depends_on('mpix-launch-swift')
    depends_on('tau', when='+tau')

    def install(self, spec, prefix):
        copy_tree('.', prefix)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Syscalc(MakefilePackage):
    """ A tool to derive theoretical systematic uncertainties"""

    homepage = "https://cp3.irmp.ucl.ac.be/projects/madgraph/wiki/SysCalc"
    url      = "http://madgraph.phys.ucl.ac.be/Downloads/SysCalc_V1.1.7.tar.gz"

    version('1.1.7', sha256='d33219a525a54733bdfd02d8314288bdecd977fb1e97ddeaf13bf470bbbdd386')

    depends_on('lhapdf@6:')

    def url_for_version(self, version):
        url = self.url.rsplit('/', 1)[0]
        url += '/SysCalc_V{0}.tar.gz'

        url = url.format(version)
        return url

    def install(self, spec, prefix):
        mkdirp(prefix.bin)
        install('sys_calc', prefix.bin)
        install_tree('include', prefix.include)
<commit_msg>Syscalc hash (#19323)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Syscalc(MakefilePackage):
    """ A tool to derive theoretical systematic uncertainties"""

    homepage = "https://cp3.irmp.ucl.ac.be/projects/madgraph/wiki/SysCalc"
    url      = "https://bazaar.launchpad.net/~mgtools/mg5amcnlo/SysCalc/tarball/17"

    version('1.1.7', sha256='ac73df0f9f195eb62601fafc2eede3db17a562750f7971616870d6df4abd1b6c',
            url='https://bazaar.launchpad.net/~mgtools/mg5amcnlo/SysCalc/tarball/17')

    depends_on('lhapdf@6:')

    def url_for_version(self, version):
        url = self.url.rsplit('/', 1)[0]
        url += '/SysCalc_V{0}.tar.gz'

        url = url.format(version)
        return url

    def install(self, spec, prefix):
        mkdirp(prefix.bin)
        install('sys_calc', prefix.bin)
        install_tree('include', prefix.include)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from contextlib import contextmanager
from collections import namedtuple

from Xlib import X, display, Xatom
from Xlib.error import XError
from Xlib.ext import randr

from edider.parser import BaseMonitor

Geometry = namedtuple('Geometry', 'x y width height')

@contextmanager
def get_window(i_screen=0):
    "Create & manage a x-window."
    screen = display.Display().screen(i_screen)
    window = screen.root.create_window(0, 0, 1, 1, 1, screen.root_depth)
    yield window
    window.destroy()

def get_connected_outputs(i_screen=0):
    "Yield the X11-index for each connected monitor"
    with get_window(i_screen) as window:
        res = randr.get_screen_resources(window)
        outputs = res.outputs
        for output in outputs:
            info = randr.get_output_info(window, output, 0)
            if info.connection == 0:
                yield output

CRTCInfo = namedtuple('CRTCInfo', ('idx', 'info'))
def crtc_info(*crtc_idx):
    with get_window() as win:
        for idx in crtc_idx:
            try:
                info = randr.get_crtc_info(win, idx, 0)._data
            except XError:      # will error if crtc == 0
                info = {}
            yield CRTCInfo(idx, info)

class X11Output(object):
    def __init__(self, idx):
        self.idx = idx

    @property
    def edid(self):
        try:
            return self._edid
        except AttributeError:
            pass
        PROPERTY_EDID = display.Display().intern_atom('EDID', only_if_exists=True)
        with get_window() as win:
            edid = randr.get_output_property(
                win,
                self.idx,
                PROPERTY_EDID,
                Xatom.INTEGER,
                0,
                400,
            )
            edid = bytes(edid.value)
            self._edid = edid
        return self._edid

    @property
    def info(self):
        try:
            return self._info
        except AttributeError:
            pass
        with get_window() as win:
            self._info = randr.get_output_info(win, self.idx, 0)._data
        return self._info

    @property
    def output_name(self):
        return self.info['name']

    @property
    def crtc(self):
        return next(crtc_info(self.info['crtc']))

    @property
    def crtcs(self):
        return list(crtc_info(*self.info['crtcs']))

    def _get_modes(self):
        with get_window() as win:
            res = randr.get_screen_resources(win)
            modes = res._data['modes']
            modes = [x._data for x in modes]
            return modes

    @property
    def modes(self):
        try:
            return self._modes
        except AttributeError:
            self._modes = self._get_modes()
            return self._modes

    @property
    def preferred_mode(self):
        npref = self.info['num_preferred']
        nmode = self.info['modes'][npref-1]
        modes = [x for x in self.modes if x['id'] == nmode]
        assert len(modes) == 1, 'There can only be one preferred mode.'
        return modes[0]

    @property
    def current_mode(self):
        modes = self._get_modes()
        try:
            mode_id = self.crtc.info['mode']
        except KeyError:
            return {}
        modes = [x for x in modes if x['id'] == mode_id]
        assert len(modes) == 1, 'There can only be one current mode.'
        return modes[0]

    def __repr__(self):
        cname = self.__class__.__name__
        return cname + '({})'.format(self.idx)

class Monitor(BaseMonitor):
    def __init__(self, index):
        """Index is the X11 index of the output"""
        self._id = index
        self._xout = X11Output(index)

    def _get_output_edid(self):
        return self._xout.edid

    def _dflt_resolution(self):
        d = self._xout.preferred_mode
        self._width_in_pixels, self._height_in_pixels = d['width'], d['height']

    @property
    def output_name(self):
        return self._xout.output_name

    @property
    def status(self):
        crtc = self._xout.crtc
        if crtc.idx == 0:
            return 'off'
        return 'on'

    @property
    def geometry(self):
        d = self._xout.crtc.info
        try:
            x, y, width, height = d['x'], d['y'], d['width'], d['height']
        except KeyError:
            x, y, width, height = 0, 0, 0, 0
        return Geometry(x, y, width, height)

    def __str__(self):
        sstr = super().__str__()
        sstr += '\t->\t{}x{}'.format(self.width_in_pixels, self.height_in_pixels)
        return sstr

def get_monitors():
    "Return a Monitor instance for each connected computer monitor."
    return [Monitor(x) for x in get_connected_outputs()]

if __name__ == '__main__':
    monitors = get_monitors()
    # monitors = [Monitor(x) for x in get_connected_outputs()]
    for mon in monitors:
        print(mon)
<commit_msg>Add x11_output_id to Monitor properties<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from contextlib import contextmanager
from collections import namedtuple

from Xlib import X, display, Xatom
from Xlib.error import XError
from Xlib.ext import randr

from edider.parser import BaseMonitor

Geometry = namedtuple('Geometry', 'x y width height')

@contextmanager
def get_window(i_screen=0):
    "Create & manage a x-window."
    screen = display.Display().screen(i_screen)
    window = screen.root.create_window(0, 0, 1, 1, 1, screen.root_depth)
    yield window
    window.destroy()

def get_connected_outputs(i_screen=0):
    "Yield the X11-index for each connected monitor"
    with get_window(i_screen) as window:
        res = randr.get_screen_resources(window)
        outputs = res.outputs
        for output in outputs:
            info = randr.get_output_info(window, output, 0)
            if info.connection == 0:
                yield output

CRTCInfo = namedtuple('CRTCInfo', ('idx', 'info'))
def crtc_info(*crtc_idx):
    with get_window() as win:
        for idx in crtc_idx:
            try:
                info = randr.get_crtc_info(win, idx, 0)._data
            except XError:      # will error if crtc == 0
                info = {}
            yield CRTCInfo(idx, info)

class X11Output(object):
    def __init__(self, idx):
        self.idx = idx

    @property
    def edid(self):
        try:
            return self._edid
        except AttributeError:
            pass
        PROPERTY_EDID = display.Display().intern_atom('EDID', only_if_exists=True)
        with get_window() as win:
            edid = randr.get_output_property(
                win,
                self.idx,
                PROPERTY_EDID,
                Xatom.INTEGER,
                0,
                400,
            )
            edid = bytes(edid.value)
            self._edid = edid
        return self._edid

    @property
    def info(self):
        try:
            return self._info
        except AttributeError:
            pass
        with get_window() as win:
            self._info = randr.get_output_info(win, self.idx, 0)._data
        return self._info

    @property
    def output_name(self):
        return self.info['name']

    @property
    def crtc(self):
        return next(crtc_info(self.info['crtc']))

    @property
    def crtcs(self):
        return list(crtc_info(*self.info['crtcs']))

    def _get_modes(self):
        with get_window() as win:
            res = randr.get_screen_resources(win)
            modes = res._data['modes']
            modes = [x._data for x in modes]
            return modes

    @property
    def modes(self):
        try:
            return self._modes
        except AttributeError:
            self._modes = self._get_modes()
            return self._modes

    @property
    def preferred_mode(self):
        npref = self.info['num_preferred']
        nmode = self.info['modes'][npref-1]
        modes = [x for x in self.modes if x['id'] == nmode]
        assert len(modes) == 1, 'There can only be one preferred mode.'
        return modes[0]

    @property
    def current_mode(self):
        modes = self._get_modes()
        try:
            mode_id = self.crtc.info['mode']
        except KeyError:
            return {}
        modes = [x for x in modes if x['id'] == mode_id]
        assert len(modes) == 1, 'There can only be one current mode.'
        return modes[0]

    def __repr__(self):
        cname = self.__class__.__name__
        return cname + '({})'.format(self.idx)

class Monitor(BaseMonitor):
    def __init__(self, index):
        """Index is the X11 index of the output"""
        self._id = index
        self._xout = X11Output(index)

    def _get_output_edid(self):
        return self._xout.edid

    def _dflt_resolution(self):
        d = self._xout.preferred_mode
        self._width_in_pixels, self._height_in_pixels = d['width'], d['height']

    @property
    def output_name(self):
        return self._xout.output_name

    @property
    def x11_output_id(self):
        return self._xout.idx

    @property
    def status(self):
        crtc = self._xout.crtc
        if crtc.idx == 0:
            return 'off'
        return 'on'

    @property
    def geometry(self):
        d = self._xout.crtc.info
        try:
            x, y, width, height = d['x'], d['y'], d['width'], d['height']
        except KeyError:
            x, y, width, height = 0, 0, 0, 0
        return Geometry(x, y, width, height)

    def __str__(self):
        sstr = super().__str__()
        sstr += '\t->\t{}x{}'.format(self.width_in_pixels, self.height_in_pixels)
        return sstr

def get_monitors():
    "Return a Monitor instance for each connected computer monitor."
    return [Monitor(x) for x in get_connected_outputs()]

if __name__ == '__main__':
    monitors = get_monitors()
    # monitors = [Monitor(x) for x in get_connected_outputs()]
    for mon in monitors:
        print(mon)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#
import subprocess
from mounts import Mounts

class FreezeError(object):
    def __init__(self):
        self.errorcode = None
        self.fstype = None
        self.path = None
    def __str__(self):
        return "errorcode:" + str(self.errorcode) + " fstype:" + str(self.fstype) + " path" + str(self.path)

class FreezeResult(object):
    def __init__(self):
        self.errors = []
    def __str__(self):
        error_str = ""
        for error in self.errors:
            error_str+=(str(error)) + "\n"
        return error_str

class FsFreezer:
    def __init__(self, patching, logger):
        """
        """
        self.patching = patching
        self.logger = logger
        self.mounts = Mounts(patching = self.patching, logger = self.logger)
        self.frozen_items = set()
        self.unfrozen_items = set()

    def freeze(self, mount):
        """
        for xfs we should use the xfs_freeze, or we just use fsfreeze
        """
        freeze_error = FreezeError()
        path = mount.mount_point

        if(path in self.frozen_items):
            self.logger.log("skipping the mount point " + str(path) + " because we already freezed it")
            freeze_return_code = 0
        else:
            self.logger.log('freeze...' + str(path) + ' type ' + str(mount.type))
            freeze_return_code = 0
            if(self.should_skip(mount)):
                self.logger.log('skip for devtmpfs and devpts ' + str(mount.type))
            else:
                self.frozen_items.add(path)
                freeze_return_code = subprocess.call(['fsfreeze', '-f', path])
            self.logger.log('freeze_result...' + str(freeze_return_code))
        freeze_error.errorcode = freeze_return_code

        if(freeze_return_code != 0):
            freeze_error.path = path
        return freeze_error

    def unfreeze(self, mount):
        """
        for xfs we should use the xfs_freeze -u, or we just use fsfreeze -u
        """
        freeze_error = FreezeError()
        path = mount.mount_point
        self.logger.log('unfreeze...' + str(path) + ' type ' + str(mount.type))
        unfreeze_return_code = 0
        if(self.should_skip(mount)):
            self.logger.log('skip for the type ' + str(mount.type))
        else:
            if(not path in self.unfrozen_items):
                freeze_return_code = 0
                self.unfrozen_items.add(path)
                if(mount.type == 'xfs'):
                    unfreeze_return_code = subprocess.call(['xfs_freeze', '-u', path])
                else:
                    unfreeze_return_code = subprocess.call(['fsfreeze', '-u', path])
            else:
                self.logger.log('the item ' + str(path) + ' is already unfreezed, so skip it')
        self.logger.log('unfreeze_result...' + str(unfreeze_return_code))
        freeze_error.errorcode = unfreeze_return_code
        if(unfreeze_return_code != 0):
            freeze_error.path = path
        return freeze_error

    def should_skip(self, mount):
        if((mount.fstype == 'ext3' or mount.fstype == 'ext4' or mount.fstype == 'xfs' or mount.fstype == 'btrfs') and mount.type != 'loop'):
            return False
        else:
            return True

    def freezeall(self):
        self.root_seen = False
        freeze_result = FreezeResult()
        for mount in self.mounts.mounts:
            if(mount.mount_point == '/'):
                self.root_seen = True
                self.root_mount = mount
            elif(mount.mount_point):
                try:
                    freezeError = self.freeze(mount)
                    if(freezeError.errorcode != 0):
                        freeze_result.errors.append(freezeError)
                except Exception, e:
                    freezeError = FreezeError()
                    freezeError.errorcode = -1
                    freezeError.path = mount.mount_point
                    freeze_result.errors.append(freezeError)
                    self.logger.log(str(e))

        if(self.root_seen):
            freezeError = self.freeze(self.root_mount)
            if(freezeError.errorcode != 0):
                freeze_result.errors.append(freezeError)
        return freeze_result

    def unfreezeall(self):
        self.root_seen = False
        unfreeze_result = FreezeResult()
        for mount in self.mounts.mounts:
            if(mount.mount_point == '/'):
                self.root_seen = True
                self.root_mount = mount
            elif(mount.mount_point):
                try:
                    freezeError = self.unfreeze(mount)
                    if(freezeError.errorcode != 0):
                        unfreeze_result.errors.append(freezeError)
                except Exception,e:
                    freezeError = FreezeError()
                    freezeError.errorcode = -1
                    freezeError.path = mount.mount_point
                    unfreeze_result.errors.append(freezeError)
        if(self.root_seen):
            freezeError = self.unfreeze(self.root_mount)
            if(freezeError.errorcode != 0):
                unfreeze_result.errors.append(freezeError)
        return unfreeze_result<commit_msg>the fsfreeze also support the xfs.<commit_after>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2014 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#
import subprocess
from mounts import Mounts

class FreezeError(object):
    def __init__(self):
        self.errorcode = None
        self.fstype = None
        self.path = None
    def __str__(self):
        return "errorcode:" + str(self.errorcode) + " fstype:" + str(self.fstype) + " path" + str(self.path)

class FreezeResult(object):
    def __init__(self):
        self.errors = []
    def __str__(self):
        error_str = ""
        for error in self.errors:
            error_str+=(str(error)) + "\n"
        return error_str

class FsFreezer:
    def __init__(self, patching, logger):
        """
        """
        self.patching = patching
        self.logger = logger
        self.mounts = Mounts(patching = self.patching, logger = self.logger)
        self.frozen_items = set()
        self.unfrozen_items = set()

    def freeze(self, mount):
        """
        for xfs we should use the xfs_freeze, or we just use fsfreeze
        """
        freeze_error = FreezeError()
        path = mount.mount_point

        if(path in self.frozen_items):
            self.logger.log("skipping the mount point " + str(path) + " because we already freezed it")
            freeze_return_code = 0
        else:
            self.logger.log('freeze...' + str(path) + ' type ' + str(mount.type))
            freeze_return_code = 0
            if(self.should_skip(mount)):
                self.logger.log('skip for devtmpfs and devpts ' + str(mount.type))
            else:
                self.frozen_items.add(path)
                freeze_return_code = subprocess.call(['fsfreeze', '-f', path])
            self.logger.log('freeze_result...' + str(freeze_return_code))
        freeze_error.errorcode = freeze_return_code

        if(freeze_return_code != 0):
            freeze_error.path = path
        return freeze_error

    def unfreeze(self, mount):
        """
        for xfs we should use the xfs_freeze -u, or we just use fsfreeze -u
        """
        freeze_error = FreezeError()
        path = mount.mount_point
        self.logger.log('unfreeze...' + str(path) + ' type ' + str(mount.type))
        unfreeze_return_code = 0
        if(self.should_skip(mount)):
            self.logger.log('skip for the type ' + str(mount.type))
        else:
            if(not path in self.unfrozen_items):
                freeze_return_code = 0
                self.unfrozen_items.add(path)
                unfreeze_return_code = subprocess.call(['fsfreeze', '-u', path])
            else:
                self.logger.log('the item ' + str(path) + ' is already unfreezed, so skip it')
        self.logger.log('unfreeze_result...' + str(unfreeze_return_code))
        freeze_error.errorcode = unfreeze_return_code
        if(unfreeze_return_code != 0):
            freeze_error.path = path
        return freeze_error

    def should_skip(self, mount):
        if((mount.fstype == 'ext3' or mount.fstype == 'ext4' or mount.fstype == 'xfs' or mount.fstype == 'btrfs') and mount.type != 'loop'):
            return False
        else:
            return True

    def freezeall(self):
        self.root_seen = False
        freeze_result = FreezeResult()
        for mount in self.mounts.mounts:
            if(mount.mount_point == '/'):
                self.root_seen = True
                self.root_mount = mount
            elif(mount.mount_point):
                try:
                    freezeError = self.freeze(mount)
                    if(freezeError.errorcode != 0):
                        freeze_result.errors.append(freezeError)
                except Exception, e:
                    freezeError = FreezeError()
                    freezeError.errorcode = -1
                    freezeError.path = mount.mount_point
                    freeze_result.errors.append(freezeError)
                    self.logger.log(str(e))

        if(self.root_seen):
            freezeError = self.freeze(self.root_mount)
            if(freezeError.errorcode != 0):
                freeze_result.errors.append(freezeError)
        return freeze_result

    def unfreezeall(self):
        self.root_seen = False
        unfreeze_result = FreezeResult()
        for mount in self.mounts.mounts:
            if(mount.mount_point == '/'):
                self.root_seen = True
                self.root_mount = mount
            elif(mount.mount_point):
                try:
                    freezeError = self.unfreeze(mount)
                    if(freezeError.errorcode != 0):
                        unfreeze_result.errors.append(freezeError)
                except Exception,e:
                    freezeError = FreezeError()
                    freezeError.errorcode = -1
                    freezeError.path = mount.mount_point
                    unfreeze_result.errors.append(freezeError)
        if(self.root_seen):
            freezeError = self.unfreeze(self.root_mount)
            if(freezeError.errorcode != 0):
                unfreeze_result.errors.append(freezeError)
        return unfreeze_result<|endoftext|>


<commit_before>import re
import codecs
import collections
import os
import polib

def write_po_header(f, lang):

    f.writelines(["# UrBackup translation\n",
                  "# Copyright (C) 2011-2014 See translators\n",
                  "# This file is distributed under the same license as the UrBackup package.\n",
                  "msgid \"\"\n",
                  "msgstr \"\"\n",
                  "\"MIME-Version: 1.0\\n\"\n",
                  "\"Content-Type: text/plain; charset=UTF-8\\n\"\n",
                  "\"Content-Transfer-Encoding: 8bit\\n\"\n",
                  "\"Language: "+lang+"\\n\"\n",
                  "\n"]);

def po_from_translations():
    with codecs.open("js/translation.js", "r", encoding="utf8") as translation_file:

        translation = translation_file.readlines();

        lang = ""

        translations = {};

        for line in translation:

            m = re.search("translations\.([A-Za-z_]*)={", line, 0)

            if m:

                lang = m.group(1)

                translations[lang] = collections.OrderedDict()

            m = re.search("\"(.*?)\".*:.*\"(.*)\"", line, 0)

            if m and len(lang)>0:

                key = m.group(1)
                val = m.group(2)

                translations[lang][key] = val

        enkeys = collections.OrderedDict()

        for lang in translations:

            with codecs.open(lang+".po" ,"w", encoding="utf8") as pofile:

                write_po_header(pofile, lang)

                trans = translations[lang]

                for key in trans:

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+trans[key]+"\"\n",
                                       "\n"])

                    enkeys[key] = True

        templ_dir = r"www\templates"
        skip = ["target_db_version", "time" ]
        for file in os.listdir(templ_dir):
            if file.endswith(".htm"):

                with codecs.open(templ_dir+"\\"+file ,"r", encoding="utf8") as templ:

                    for line in templ.readlines():

                        for m in re.finditer("{t([^\|]*?)(\|.*?)?}", line):

                            key = m.group(1)

                            if key not in enkeys and (not m.group(2) or "s" not in m.group(2)):

                                if "t"+key not in skip:
                                    enkeys["t"+key] = True

        with codecs.open("en.po" ,"w", encoding="utf8") as pofile:

            write_po_header(pofile, "en")

            for key in enkeys:
                if key in translations["en"]:

                    trans = translations["en"][key];

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+trans+"\"\n",
                                       "\n"])

                elif len(key)>0 and key[0]=="t":

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+key[1:]+"\"\n",
                                       "\n"])

def translations_from_po():

    lang_map = { "fa_IR": "fa" }

    with codecs.open("js/translation.js", "w", encoding="utf8") as output:

        output.write("if(!window.translations) translations=new Object();\n")

        tdir = "translations/urbackup.webinterface"

        prev_lang = False

        for file in os.listdir(tdir):
            if file.endswith(".po"):

                m = re.search("([A-Za-z_]*).*\.po", file);

                if m:
                    lang = m.group(1)

                    po = polib.pofile(tdir + "/" + file)

                    if len(po.translated_entries())>0:

                        if prev_lang:
                            output.write("\n}\n")

                        prev_lang = True

                        if lang in lang_map:
                            lang = lang_map[lang]

                        output.write("translations."+lang+" = {\n")

                        prev_entry = False

                        for entry in po.translated_entries():

                            if prev_entry:
                                output.write(",\n")

                            prev_entry = True

                            msgid = entry.msgid.replace("\"", "\\\"")
                            msgstr = entry.msgstr.replace("\"", "\\\"")
                            msgstr = msgstr.replace("\n", "\\n")

                            output.write("\""+msgid+"\": \""+msgstr+"\"")

        if prev_lang:
            output.write("\n}\n")

translations_from_po()

                        <commit_msg>Fix translation helper script<commit_after>import re
import codecs
import collections
import os
import polib

def write_po_header(f, lang):

    f.writelines(["# UrBackup translation\n",
                  "# Copyright (C) 2011-2014 See translators\n",
                  "# This file is distributed under the same license as the UrBackup package.\n",
                  "msgid \"\"\n",
                  "msgstr \"\"\n",
                  "\"MIME-Version: 1.0\\n\"\n",
                  "\"Content-Type: text/plain; charset=UTF-8\\n\"\n",
                  "\"Content-Transfer-Encoding: 8bit\\n\"\n",
                  "\"Language: "+lang+"\\n\"\n",
                  "\n"]);

def po_from_translations():
    with codecs.open("js/translation.js", "r", encoding="utf8") as translation_file:

        translation = translation_file.readlines();

        lang = ""

        translations = {};

        for line in translation:

            m = re.search("translations\.([A-Za-z_]*)[ ]*=[ ]*{", line, 0)

            if m:

                lang = m.group(1)

                translations[lang] = collections.OrderedDict()

            m = re.search("\"(.*?)\".*:.*\"(.*)\"", line, 0)

            if m and len(lang)>0:

                key = m.group(1)
                val = m.group(2)

                translations[lang][key] = val

        enkeys = collections.OrderedDict()

        for lang in translations:

            with codecs.open(lang+".po" ,"w", encoding="utf8") as pofile:

                write_po_header(pofile, lang)

                trans = translations[lang]

                for key in trans:

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+trans[key]+"\"\n",
                                       "\n"])

                    enkeys[key] = True

        templ_dir = r"templates"
        skip = ["target_db_version", "time" ]
        for file in os.listdir(templ_dir):
            if file.endswith(".htm"):

                with codecs.open(templ_dir+"\\"+file ,"r", encoding="utf8") as templ:

                    for line in templ.readlines():

                        for m in re.finditer("{t([^\|]*?)(\|.*?)?}", line):

                            key = m.group(1)

                            if key not in enkeys and (not m.group(2) or "s" not in m.group(2)):

                                if "t"+key not in skip:
                                    enkeys["t"+key] = True

        with codecs.open("en.po" ,"w", encoding="utf8") as pofile:

            write_po_header(pofile, "en")

            for key in enkeys:
                if key in translations["en"]:

                    trans = translations["en"][key];

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+trans+"\"\n",
                                       "\n"])

                elif len(key)>0 and key[0]=="t":

                    pofile.writelines(["msgid \""+key+"\"\n",
                                       "msgstr \""+key[1:]+"\"\n",
                                       "\n"])

def translations_from_po():

    lang_map = { "fa_IR": "fa" }

    with codecs.open("js/translation.js", "w", encoding="utf8") as output:

        output.write("if(!window.translations) translations=new Object();\n")

        tdir = "translations/urbackup.webinterface"

        prev_lang = False

        for file in os.listdir(tdir):
            if file.endswith(".po"):

                m = re.search("([A-Za-z_]*).*\.po", file);

                if m:
                    lang = m.group(1)

                    po = polib.pofile(tdir + "/" + file)

                    if len(po.translated_entries())>0:

                        if prev_lang:
                            output.write("\n}\n")

                        prev_lang = True

                        if lang in lang_map:
                            lang = lang_map[lang]

                        output.write("translations."+lang+" = {\n")

                        prev_entry = False

                        for entry in po.translated_entries():

                            if prev_entry:
                                output.write(",\n")

                            prev_entry = True

                            msgid = entry.msgid.replace("\"", "\\\"")
                            msgstr = entry.msgstr.replace("\"", "\\\"")
                            msgstr = msgstr.replace("\n", "\\n")

                            output.write("\""+msgid+"\": \""+msgstr+"\"")

        if prev_lang:
            output.write("\n}\n")

translations_from_po()

                        <|endoftext|>


<commit_before>__author__ = 'Per'

# decrypts a (binary/Ascii) number with XOR-operator and given cypher

plaintext = "001100"  # STRING!!!     will become input later
plainKey = "0110"
decryptedText = ""

def checkIfBinaryIsUsed(key, plaintext):  # checks if text and key is in binary code (made of only 1's and 0's)

    textNumbersAreCorrect = None  # if text is binary => true, else => false
    keyNumbersAreCorrect = None  # if key is binary => true, else => false
    allNumbersAreCorrect = None

    for char in key:   # checks on Key

        if (char == "0") or (char == "1"):
            keyNumbersAreCorrect = True
        else:
            keyNumbersAreCorrect = False
            break

    for char in plaintext:  # checks on Text

        if (char == "0") or (char == "1"):
            textNumbersAreCorrect = True
        else:
            textNumbersAreCorrect = False
            break

    if (keyNumbersAreCorrect == True) & (textNumbersAreCorrect == True):  # add's both checks to 1

        allNumbersAreCorrect = True
    else:
        allNumbersAreCorrect = False

    return allNumbersAreCorrect

def matchKeyLengthToPlaintextLenght(key, plaintext):  # only works if binary_only numbers are used in key and plaintext! elif returns None
    matchedKey = None
    onlyBinaryUsed = checkIfBinaryIsUsed(key, plaintext)
    if onlyBinaryUsed == True:
        matchedKey = key  # for case if length of key and text is the same

        if len(plaintext) > len(key):   # if key is to short

            lengthDiff = len(plaintext) - len(key)  # the length difference between key and plaintext
            matchedKeyAsList = []

            for letter in key:  # makes key a list

                matchedKeyAsList.append(letter)

            matchedKeyAsList.append(matchedKeyAsList[0])  # and add's it's first letter to the end

            if lengthDiff > 1:   # if more than one letter needs to be added to the key

                for i in range (1, lengthDiff):  # it is appended to the end of the list

                    matchedKeyAsList.append(matchedKeyAsList[i])

            matchedKey = "".join(matchedKeyAsList)   # joins key list back to String

        elif len(plaintext) < len(key):   # if key is to long

            lengthDiff = len(key) - len(plaintext)  # difference between key and text
            matchedKeyAsList = []

            for letter in key:  # makes key a list

                matchedKeyAsList.append(letter)

            for i in range (0, lengthDiff):  # and removes as many items from 0 to length diff, so length of text and key is equal
                matchedKeyAsList.pop(i)

            matchedKey = "".join(matchedKeyAsList)  # joins key list back to String

    print("Only binary is used: " , onlyBinaryUsed)
    return matchedKey

matchedKey = matchKeyLengthToPlaintextLenght(plainKey, plaintext)

print(matchedKey)

<commit_msg>encrypting with XOR now works -> decrypt (= encrypt encryptedText) also working<commit_after>__author__ = 'Per'

# decrypts a (binary/Ascii) number with XOR-operator and given cypher

plaintext = "001100"  # STRING!!!     will become input later
plainKey = "0110"

def checkIfBinaryIsUsed(key, plaintext):  # checks if text and key is in binary code (made of only 1's and 0's)

    textNumbersAreCorrect = None  # if text is binary => true, else => false
    keyNumbersAreCorrect = None  # if key is binary => true, else => false
    allNumbersAreCorrect = None

    for char in key:   # checks on Key

        if (char == "0") or (char == "1"):
            keyNumbersAreCorrect = True
        else:
            keyNumbersAreCorrect = False
            break

    for char in plaintext:  # checks on Text

        if (char == "0") or (char == "1"):
            textNumbersAreCorrect = True
        else:
            textNumbersAreCorrect = False
            break

    if (keyNumbersAreCorrect == True) & (textNumbersAreCorrect == True):  # add's both checks to 1

        allNumbersAreCorrect = True
    else:
        allNumbersAreCorrect = False

    return allNumbersAreCorrect

def matchKeyLengthToPlaintextLenght(key, plaintext):  # only works if binary_only numbers are used in key and plaintext! elif returns None
    matchedKey = None
    onlyBinaryUsed = checkIfBinaryIsUsed(key, plaintext)
    if onlyBinaryUsed == True:
        matchedKey = key  # for case if length of key and text is the same

        if len(plaintext) > len(key):   # if key is to short

            lengthDiff = len(plaintext) - len(key)  # the length difference between key and plaintext
            matchedKeyAsList = []

            for letter in key:  # makes key a list

                matchedKeyAsList.append(letter)

            matchedKeyAsList.append(matchedKeyAsList[0])  # and add's it's first letter to the end

            if lengthDiff > 1:   # if more than one letter needs to be added to the key

                for i in range (1, lengthDiff):  # it is appended to the end of the list

                    matchedKeyAsList.append(matchedKeyAsList[i])

            matchedKey = "".join(matchedKeyAsList)   # joins key list back to String

        elif len(plaintext) < len(key):   # if key is to long

            lengthDiff = len(key) - len(plaintext)  # difference between key and text
            matchedKeyAsList = []

            for letter in key:  # makes key a list

                matchedKeyAsList.append(letter)

            for i in range (0, lengthDiff):  # and removes as many items from 0 to length diff, so length of text and key is equal
                matchedKeyAsList.pop(i)

            matchedKey = "".join(matchedKeyAsList)  # joins key list back to String

    print("Only binary is used: " , onlyBinaryUsed)
    return matchedKey

matchedKey = matchKeyLengthToPlaintextLenght(plainKey, plaintext)

def encryptWithXor(matchedKey, plaintext):
    counter = 0
    encryptedTextAsList = []
    for char in plaintext:
        encryptedChar = ""
        charInKey = matchedKey[counter]
        if (char == "0") & (charInKey == "0"):
            encryptedChar = "0"
        elif (char == "0") & (charInKey == "1"):
            encryptedChar = "1"
        elif (char == "1") & (charInKey == "0"):
            encryptedChar = "1"
        elif (char == "1") & (charInKey == "1"):
            encryptedChar = "0"
        encryptedTextAsList.append(encryptedChar)
        counter += 1
    encryptedText = "".join(encryptedTextAsList)
    return encryptedText

encryptedText = encryptWithXor(matchedKey, plaintext)
print(encryptedText)

decryptedText = encryptWithXor(matchedKey, encryptedText)
print(decryptedText)<|endoftext|>


<commit_before>""" this module impls the logic for 'bru make', which builds the downloaded
   modules via gyp and local compiler toolchain
"""

import os
import re
import glob
import platform
import brulib.install

def cmd_make(config, verbose):
    """ this command makes some educated guesses about which toolchain
        the user probably wants to run, then invokes gyp to create the
        makefiles for this toolchain and invokes the build. On Linux
        'bru make' is likely equivalent to:
           >gyp *gyp --depth=.
           >make
        On Windows it's likely equivalent to:
           >gyp --depth=. package.gyp -G msvs_version=2012
           >C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe package.sln
        The main purpose of 'bru make' is really to spit out these two
        command lines as a quick reminder for how to build via cmd line.
        param config contains 'Release' or 'Debug'
        param verbose 0 means not verbose, >= 1 means higher verbosity level
            (whatever that means in the underlying toolchain)
    """
    print("running 'bru make --config {}'".format(config))

    # first locate the single gyp in the cwd
    bru_file = brulib.install.get_single_bru_file('.')
    if bru_file == None:
        raise Exception("there's no *.bru file in current work dir, "
            'e.g. run "bru install googlemock" first to create one')
    gyp_file = bru_file[:-3] + 'gyp'
    if not os.path.exists(gyp_file):
        raise Exception(bru_file,'has no companion *.gyp file, '
            'e.g. recreate one via "bru install googlemock"')

    system = platform.system()
    if system == 'Windows':
        cmd_make_win(gyp_file, config)
    elif system == 'Linux':
        cmd_make_linux(gyp_file, config, verbose)
    else:
        raise Exception('no idea how to invoke gyp & toolchain on platform {}'\
            .format(system))

def get_latest_msbuild_exe():
    """ return path to latest msbuild on Windows machine """
    env = os.environ
    windir = env['SystemRoot'] if 'SystemRoot' in env else env['windir']
    glob_expr = os.path.join(windir, 'Microsoft.NET', 'Framework',
        '**', 'msbuild.exe')
    msbuilds = glob.glob(glob_expr)
    return max(msbuilds)  # not alphanumeric, should be good enough tho

def get_latest_installed_msvs_version():
    """ e.g. return 110 (aka VC11 aka VS 2012)) if msvs 2012 is installed.
        If multiple vs versions are installed then pick latest.
        Return None if no installs are found
    """
    # whats a good way to detect the msvs version?
    # a) scan for install dirs like
    #    c:\Program Files (x86)\Microsoft Visual Studio 10.0
    # b) scan for env vars like VS110COMNTOOLS
    # Let's do (b) for now.
    # See also https://code.google.com/p/gyp/source/browse/trunk/pylib/gyp/MSVSVersion.py
    msvs_versions = []
    regex = re.compile('^VS([0-10]+)COMNTOOLS$')
    for key in os.environ:
        match = regex.match(key)
        if match != None:
            msvs_versions.append(int(match.group(1)))
    if len(msvs_versions) == 0:
        return None
    latest = max(msvs_versions) # e.g. 110
    if len(msvs_versions) > 1:
        print('detected installs of msvs {}, choosing latest {}'.format(
            msvs_versions, latest))
    return latest

def get_latest_installed_msvs_year():
    """ e.g. return 2012 (aka VC11) if msvs 2012 is installed. If multiple
        vs versions are installed then pick latest.
        Return None if no installs are found
    """
    latest = get_latest_installed_msvs_version()
    if latest == None:
        return None
    msvs_version2year = {
        80: 2005,
        90: 2008,
        100: 2010,
        110: 2012,
    }
    if not latest in msvs_version2year:
        print('not sure how to map VC{} to a VS year, defaulting to VS 2012'
            .format(latest))
        return 2012
    return msvs_version2year[latest]

def run_gyp(gyp_cmdline):
    print("running '{}'".format(gyp_cmdline))
    returncode = os.system(gyp_cmdline)
    if returncode != 0:
        raise Exception('error running gyp, did you install it?'
            ' Instructions at https://github.com/KjellSchubert/bru')

def cmd_make_win(gyp_filename, config):
    # TODO: locate msvs version via glob
    msvs_version = get_latest_installed_msvs_year()
    if msvs_version == None:
        print('WARNING: no msvs installation detected, did you install it? '
            'Defaulting to msvs 2012.')
        msvs_version = 2012
    gyp_cmdline = 'gyp --depth=. {} -G msvs_version={}'.format(
        gyp_filename, msvs_version)
    run_gyp(gyp_cmdline)
    # gyp should have created a *.sln file, verify that.
    # if it didnt that pass a msvc generator option to gyp in a more explicit
    # fashion (is -G msvs_version enough? need GYP_GENERATORS=msvs?).
    sln_filename = gyp_filename[:-3] + 'sln'
    if not os.path.exists(sln_filename):
        raise Exception('gyp unexpectedly did not generate a *.sln file, '
            'you may wanna invoke gyp manually to generate the expected '
            'make/sln/ninja files, e.g. set GYP_GENERATORS=msvs')

    # there are many ways to build the *.sln now, e.g. pass it to devenv
    # or alternatively to msbuild. Lets do msbuild here:
    # TODO locate msbuild via glob
    msbuild_exe = get_latest_msbuild_exe()
    if msbuild_exe == None:
        raise Exception('did not detect any installs of msbuild, these should'
            ' be part of .NET installations, please install msbuild or .NET')
    msbuild_cmdline = '{} {} /p:Configuration={}'.format(
        msbuild_exe, sln_filename, config)
    print("running msvs via msbuild: '{}'".format(msbuild_cmdline))
    returncode = os.system(msbuild_cmdline)
    if returncode != 0:
        raise Exception('msbuild failed with errors, returncode =', returncode)
    print('Build complete.')

def cmd_make_linux(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)

    gyp_cmdline = 'gyp --depth=. {}'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    if not os.path.exists('Makefile'):
        raise Exception('gyp did not generate ./Makefile, no idea how to '
            'build with your toolchain, please build manually')
    make_cmdline = 'make BUILDTYPE={} V={}'.format(config,
            '1' if verbose >= 1 else '')
    print("running '{}'".format(make_cmdline))
    returncode = os.system(make_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')
<commit_msg>less spammy/verbose msbuild output<commit_after>""" this module impls the logic for 'bru make', which builds the downloaded
   modules via gyp and local compiler toolchain
"""

import os
import re
import sys
import glob
import platform
import brulib.install

def cmd_make(config, verbose):
    """ this command makes some educated guesses about which toolchain
        the user probably wants to run, then invokes gyp to create the
        makefiles for this toolchain and invokes the build. On Linux
        'bru make' is likely equivalent to:
           >gyp *gyp --depth=.
           >make
        On Windows it's likely equivalent to:
           >gyp --depth=. package.gyp -G msvs_version=2012
           >C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild.exe package.sln
        The main purpose of 'bru make' is really to spit out these two
        command lines as a quick reminder for how to build via cmd line.
        param config contains 'Release' or 'Debug'
        param verbose 0 means not verbose, >= 1 means higher verbosity level
            (whatever that means in the underlying toolchain)
    """
    print("running 'bru make --config {}'".format(config))

    # first locate the single gyp in the cwd
    bru_file = brulib.install.get_single_bru_file('.')
    if bru_file == None:
        raise Exception("there's no *.bru file in current work dir, "
            'e.g. run "bru install googlemock" first to create one')
    gyp_file = bru_file[:-3] + 'gyp'
    if not os.path.exists(gyp_file):
        raise Exception(bru_file,'has no companion *.gyp file, '
            'e.g. recreate one via "bru install googlemock"')

    system = platform.system()
    if system == 'Windows':
        cmd_make_win(gyp_file, config)
    elif system == 'Linux':
        cmd_make_linux(gyp_file, config, verbose)
    else:
        raise Exception('no idea how to invoke gyp & toolchain on platform {}'\
            .format(system))

def get_latest_msbuild_exe():
    """ return path to latest msbuild on Windows machine """
    env = os.environ
    windir = env['SystemRoot'] if 'SystemRoot' in env else env['windir']
    glob_expr = os.path.join(windir, 'Microsoft.NET', 'Framework',
        '**', 'msbuild.exe')
    msbuilds = glob.glob(glob_expr)
    return max(msbuilds)  # not alphanumeric, should be good enough tho

def get_latest_installed_msvs_version():
    """ e.g. return 110 (aka VC11 aka VS 2012)) if msvs 2012 is installed.
        If multiple vs versions are installed then pick latest.
        Return None if no installs are found
    """
    # whats a good way to detect the msvs version?
    # a) scan for install dirs like
    #    c:\Program Files (x86)\Microsoft Visual Studio 10.0
    # b) scan for env vars like VS110COMNTOOLS
    # Let's do (b) for now.
    # See also https://code.google.com/p/gyp/source/browse/trunk/pylib/gyp/MSVSVersion.py
    msvs_versions = []
    regex = re.compile('^VS([0-10]+)COMNTOOLS$')
    for key in os.environ:
        match = regex.match(key)
        if match != None:
            msvs_versions.append(int(match.group(1)))
    if len(msvs_versions) == 0:
        return None
    latest = max(msvs_versions) # e.g. 110
    if len(msvs_versions) > 1:
        print('detected installs of msvs {}, choosing latest {}'.format(
            msvs_versions, latest))
    return latest

def get_latest_installed_msvs_year():
    """ e.g. return 2012 (aka VC11) if msvs 2012 is installed. If multiple
        vs versions are installed then pick latest.
        Return None if no installs are found
    """
    latest = get_latest_installed_msvs_version()
    if latest == None:
        return None
    msvs_version2year = {
        80: 2005,
        90: 2008,
        100: 2010,
        110: 2012,
    }
    if not latest in msvs_version2year:
        print('not sure how to map VC{} to a VS year, defaulting to VS 2012'
            .format(latest))
        return 2012
    return msvs_version2year[latest]

def run_gyp(gyp_cmdline):
    print("running '{}'".format(gyp_cmdline))
    returncode = os.system(gyp_cmdline)
    if returncode != 0:
        raise Exception('error running gyp, did you install it?'
            ' Instructions at https://github.com/KjellSchubert/bru')

def cmd_make_win(gyp_filename, config):
    # TODO: locate msvs version via glob
    msvs_version = get_latest_installed_msvs_year()
    if msvs_version == None:
        print('WARNING: no msvs installation detected, did you install it? '
            'Defaulting to msvs 2012.')
        msvs_version = 2012
    gyp_cmdline = 'gyp --depth=. {} -G msvs_version={}'.format(
        gyp_filename, msvs_version)
    run_gyp(gyp_cmdline)
    # gyp should have created a *.sln file, verify that.
    # if it didnt that pass a msvc generator option to gyp in a more explicit
    # fashion (is -G msvs_version enough? need GYP_GENERATORS=msvs?).
    sln_filename = gyp_filename[:-3] + 'sln'
    if not os.path.exists(sln_filename):
        raise Exception('gyp unexpectedly did not generate a *.sln file, '
            'you may wanna invoke gyp manually to generate the expected '
            'make/sln/ninja files, e.g. set GYP_GENERATORS=msvs')

    # there are many ways to build the *.sln now, e.g. pass it to devenv
    # or alternatively to msbuild. Lets do msbuild here:
    # TODO locate msbuild via glob
    msbuild_exe = get_latest_msbuild_exe()
    if msbuild_exe == None:
        raise Exception('did not detect any installs of msbuild, these should'
            ' be part of .NET installations, please install msbuild or .NET')
    # TODO? add /toolsversion:<version> ?
    msbuild_cmdline = '{} {} /p:Configuration={} /verbosity:minimal'.format(
        msbuild_exe, sln_filename, config)
    print("running msvs via msbuild: '{}'".format(msbuild_cmdline))
    sys.stdout.flush() # otherwise msbuild log lines show up first
    returncode = os.system(msbuild_cmdline)
    if returncode != 0:
        raise Exception('msbuild failed with errors, returncode =', returncode)
    print('Build complete.')

def cmd_make_linux(gyp_filename, config, verbose):
    # Here we could check if ninja or some such is installed to generate ninja
    # project files. But for simplicity's sake let's just use whatever gyp
    # defaults to.

    # For some odd reason passing './package.gyp' as a param to gyp will
    # generate garbage, instead you gotta pass 'package.gyp'. Se let's
    # explicitly remove a leading ./
    dirname = os.path.dirname(gyp_filename)
    assert dirname == '.' or len(dirname) == 0
    gyp_filename = os.path.basename(gyp_filename)

    gyp_cmdline = 'gyp --depth=. {}'.format(gyp_filename)
    run_gyp(gyp_cmdline)
    if not os.path.exists('Makefile'):
        raise Exception('gyp did not generate ./Makefile, no idea how to '
            'build with your toolchain, please build manually')
    make_cmdline = 'make BUILDTYPE={} V={}'.format(config,
            '1' if verbose >= 1 else '')
    print("running '{}'".format(make_cmdline))
    returncode = os.system(make_cmdline)
    if returncode != 0:
        raise Exception('Build failed: make returned', returncode)
    print('Build complete.')
<|endoftext|>


<commit_before>import numpy as np
import pickle as pkl
from timeit import default_timer

def complement(seq):
    complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
    complseq = [complement[base] for base in seq]
    return complseq

def reverse_complement(seq):
    seq = list(seq)
    seq.reverse()
    return ''.join(complement(seq))

def get_seq_for_circularRNA(circbase, whole_seq):
    fp = open(circbase, 'r')
    for line in fp:
        values = line.split()
        chr_n = values[0]
        start = int(values[1])
        end = int(values[2])
        circurna_name = values[3]
        strand = values[5]
        seq = whole_seq[chr_n]
        extract_seq = seq[start: end]
        extract_seq = extract_seq.upper()
        if strand == '-':
            extract_seq =  reverse_complement(extract_seq)
    fp.close()

def get_hg19_whole_seq():
    from Bio import SeqIO
    with open('hg19.fa') as f:
        whole_seq = SeqIO.to_dict(SeqIO.parse(f, 'fasta'))
        for k in whole_seq:
            whole_seq[k] = whole_seq[k].seq
    return whole_seq

def feature_base(l):
    index = {'A' : 0, 'G' : 1, 'C' : 2, 'T' : 3}
    seq_tag = [index[c] for c in l.seq]
    feature = np.zeros([l.length, 4], dtype=np.float32)
    feature[np.arange(0, l.length), seq_tag] = 1
    return feature

class Loci:
    ''' A pair of positions in DNA sequence'''
    @property
    def length(self):
        return self.end - self.start

    @property
    def feature_dim(self):
        if not hasattr(self, 'features'):
            self.get_feature()
        return self.features.shape[1]

    whole_seq = None

    def init_seq(self):
        if Loci.whole_seq is None:
            Loci.whole_seq = get_hg19_whole_seq()

        seq = self.whole_seq[self.chr]
        extract_seq = seq[self.start: self.end]
        extract_seq = extract_seq.upper()
        if extract_seq.find('N') != -1:
            self.seq = None
        else:
            if self.strand == '-':
                extract_seq =  reverse_complement(extract_seq)
            self.seq = extract_seq

    def get_feature(self):
        extract_func = [feature_base]
        features = [ func(self) for func in extract_func]
        self.features = np.concatenate(features, axis = 1)

def read_sequence(file_name, label):
    with open(file_name, 'r') as f:
        locis = []
        for line in f.readlines():
            tokens = line.split()
            if len(tokens) > 3:
                l = Loci()
                l.chr = tokens[0]
                l.start = int(tokens[1])
                l.end = int(tokens[2])
                l.strand = tokens[5 if len(tokens) > 6 else 3]
                l.label = label
                locis.append(l)
        return locis

def loci_filter(l):
    return 500 <= l.length < 3000

def read_data():
    T0 = default_timer()
    try:
        with open('filt_locis.bin', 'rb') as f:
            locis = pkl.load(f)
    except IOError:
        locis = read_sequence('hsa_hg19_Rybak2015.bed', 1)
        locis.extend(read_sequence('all_exons.bed', 0))

        # locis = filter(loci_filter, locis)
        for l in locis:
            l.init_seq()

        with open('filt_locis.bin', 'wb') as f:
            pkl.dump(locis, f)
    print('read data used %.3fs' % (default_timer() - T0))
    return locis

def process_feature():
    T0 = default_timer()

    locis = read_data()
    for l in locis:
        l.get_feature()

    print('process_feature used %.3fs' % (default_timer() - T0))

    return locis
<commit_msg>Add Chrom Position Module for fast Interval Finding & Filtered Negative Examples so that those intersected with positive is removed<commit_after>import numpy as np
import pickle as pkl
from timeit import default_timer
from operator import attrgetter, itemgetter
from itertools import groupby, filterfalse
import math
import bisect
from tqdm import tqdm

def complement(seq):
    complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}
    complseq = [complement[base] for base in seq]
    return complseq

def reverse_complement(seq):
    seq = list(seq)
    seq.reverse()
    return ''.join(complement(seq))

def get_seq_for_circularRNA(circbase, whole_seq):
    fp = open(circbase, 'r')
    for line in fp:
        values = line.split()
        chr_n = values[0]
        start = int(values[1])
        end = int(values[2])
        circurna_name = values[3]
        strand = values[5]
        seq = whole_seq[chr_n]
        extract_seq = seq[start: end]
        extract_seq = extract_seq.upper()
        if strand == '-':
            extract_seq =  reverse_complement(extract_seq)
    fp.close()

def get_hg19_whole_seq():
    from Bio import SeqIO
    with open('hg19.fa') as f:
        whole_seq = SeqIO.to_dict(SeqIO.parse(f, 'fasta'))
        for k in whole_seq:
            whole_seq[k] = whole_seq[k].seq
    return whole_seq

def feature_base(l):
    index = {'A' : 0, 'G' : 1, 'C' : 2, 'T' : 3}
    seq_tag = [index[c] for c in l.seq]
    feature = np.zeros([l.length, 4], dtype=np.float32)
    feature[np.arange(0, l.length), seq_tag] = 1
    return feature

class Loci:
    ''' A pair of positions in DNA sequence'''
    @property
    def length(self):
        return self.end - self.start

    @property
    def feature_dim(self):
        if not hasattr(self, 'features'):
            self.get_feature()
        return self.features.shape[1]

    @property
    def chrom_strand(self):
        return self.chr + self.strand

    whole_seq = None

    def init_seq(self):
        if Loci.whole_seq is None:
            Loci.whole_seq = get_hg19_whole_seq()

        seq = self.whole_seq[self.chr]
        extract_seq = seq[self.start : self.end]
        extract_seq = extract_seq.upper()
        if extract_seq.find('N') != -1:
            self.seq = None
        else:
            if self.strand == '-':
                extract_seq =  reverse_complement(extract_seq)
            self.seq = extract_seq

    def get_feature(self):
        extract_func = [feature_base]
        features = [ func(self) for func in extract_func]
        self.features = np.concatenate(features, axis = 1)

def read_sequence(file_name, label):
    with open(file_name, 'r') as f:
        locis = []
        for line in f.readlines():
            tokens = line.split()
            if len(tokens) > 3:
                l = Loci()
                l.chr = tokens[0]
                l.start = int(tokens[1])
                l.end = int(tokens[2])
                l.strand = tokens[5 if len(tokens) > 6 else 3]
                l.label = label
                locis.append(l)
        return locis

class ChromPositions:
    @staticmethod
    def group_locis(locis):
        return groupby(sorted(map(lambda l : (l.chrom_strand, l), locis), key=itemgetter(0)), key=itemgetter(0))

    def __init__(self, locis):
        self.chrom_positions = dict()

        for chrom, locis in self.group_locis(locis):
            assert chrom not in self.chrom_positions
            start_positions = []
            end_positions = []
            start, end = 0, -math.inf
            for l in sorted(map(itemgetter(1), locis), key=attrgetter('start')):
                if end < l.start:
                    if end > 0:
                        start_positions.append(start)
                        end_positions.append(end)
                    start, end = l.start, l.end
                else:
                    end = max(end, l.end)
            if end > 0:
                start_positions.append(start)
                end_positions.append(end)

            self.chrom_positions[chrom] = (start_positions, end_positions)

    def get_intersection(self, loci):
        start_positions, end_positions = self.chrom_positions.get(loci.chrom_strand, ([], []))
        lp = bisect.bisect_right(end_positions, loci.start)
        rp = bisect.bisect_left(start_positions, loci.end)
        return list(zip(start_positions[lp:rp], end_positions[lp:rp]))

    def is_intersect(self, loci):
        start_positions, end_positions = self.chrom_positions.get(loci.chrom_strand, ([], []))
        lp = bisect.bisect_right(end_positions, loci.start)
        rp = bisect.bisect_left(start_positions, loci.end)
        return lp < rp

def filter_negative(locis_pos, locis_neg):
    positions = ChromPositions(locis_pos)
    return filterfalse(lambda l : positions.is_intersect(l), locis_neg)

def read_data():
    T0 = default_timer()
    try:
        with open('filt_locis.bin', 'rb') as f:
            locis = pkl.load(f)
    except IOError:
        locis = read_sequence('hsa_hg19_Rybak2015.bed', 1)
        locis_neg = read_sequence('all_exons.bed', 0)
        lp = len(locis)
        locis.extend(filter_negative(locis, locis_neg))
        ln = len(locis) - lp
        print(lp, ln)

        for l in tqdm(locis):
            l.init_seq()

        with open('filt_locis.bin', 'wb') as f:
            pkl.dump(locis, f)
    print('read data used %.3fs' % (default_timer() - T0))
    return locis

def process_feature():
    '''Deprecated'''
    T0 = default_timer()

    locis = read_data()
    for l in locis:
        l.get_feature()

    print('process_feature used %.3fs' % (default_timer() - T0))

    return locis
<|endoftext|>