<commit_before>from sympy.external import import_module

theano = import_module('theano')
if theano:
    ts = theano.scalar
    tt = theano.tensor
    xt, yt, zt = [tt.scalar(name, 'floatX') for name in 'xyz']
else:
    #bin/test will not execute any tests now
    disabled = True

import sympy
from sympy import S
sy = sympy
from sympy.abc import x, y, z, a, b, c
from sympy.printing.theanocode import (theano_code, dim_handling,
        theano_function)

def theq(a, b):
    """ theano equality """
    astr = theano.printing.debugprint(a, file='str')
    bstr = theano.printing.debugprint(b, file='str')

    if not astr == bstr:
        print
        print astr
        print bstr

    return astr == bstr

def test_symbol():
    xt = theano_code(x)
    assert isinstance(xt, (tt.TensorVariable, ts.ScalarVariable))
    assert xt.name == x.name

    assert theano_code(x, broadcastables={x: (False,)}).broadcastable == (False,)
    assert theano_code(x, broadcastables={x: (False,)}).name == x.name

def test_add():
    expr = x + y
    comp = theano_code(expr)
    assert comp.owner.op == theano.tensor.add

    comp = theano_code(expr, broadcastables={x: (False,), y: (False,)})
    assert comp.broadcastable == (False,)

    comp = theano_code(expr, broadcastables={x: (False, True), y: (False, False)})
    assert comp.broadcastable == (False, False)

def test_trig():
    assert theq(theano_code(sympy.sin(x)), tt.sin(xt))
    assert theq(theano_code(sympy.tan(x)), tt.tan(xt))

def test_many():
    expr = sy.exp(x**2 + sy.cos(y)) * sy.log(2*z)
    comp = theano_code(expr)
    expected = tt.exp(xt**2 + tt.cos(yt)) * tt.log(2*zt)
    # assert theq(comp, expected)

def test_dtype():
    assert theano_code(x, dtypes={x: 'float32'}).type.dtype == 'float32'
    assert theano_code(x, dtypes={x: 'float64'}).type.dtype == 'float64'
    assert theano_code(x+1, dtypes={x: 'float32'}).type.dtype == 'float32'
    assert theano_code(x+y, dtypes={x: 'float64', y: 'float32'}).type.dtype == 'float64'

def test_MatrixSymbol():
    X = sympy.MatrixSymbol('X', 4, 5)
    Xt = theano_code(X)
    assert isinstance(Xt, tt.TensorVariable)
    assert Xt.broadcastable == (False, False)

def test_MatMul():
    X = sympy.MatrixSymbol('X', 4, 4)
    Y = sympy.MatrixSymbol('X', 4, 4)
    Z = sympy.MatrixSymbol('X', 4, 4)
    expr = X*Y*Z
    assert isinstance(theano_code(expr).owner.op, tt.Dot)

def test_Transpose():
    X = sympy.MatrixSymbol('X', 4, 4)
    assert isinstance(theano_code(X.T).owner.op, tt.DimShuffle)

def test_MatAdd():
    X = sympy.MatrixSymbol('X', 4, 4)
    Y = sympy.MatrixSymbol('X', 4, 4)
    Z = sympy.MatrixSymbol('X', 4, 4)
    expr = X+Y+Z
    assert isinstance(theano_code(expr).owner.op, tt.Elemwise)

def test_symbols_are_created_once():
    expr = x**x
    comp = theano_code(expr)

    assert theq(comp, xt**xt)

def test_dim_handling():
    assert dim_handling([x], dim=2) == {x: (False, False)}
    assert dim_handling([x, y], dims={x: 1, y: 2}) == {x: (False, True),
                                                       y: (False, False)}
    assert dim_handling([x], broadcastables={x: (False,)}) == {x: (False,)}

def test_Rationals():
    assert theq(theano_code(sympy.Integer(2) / 3), tt.true_div(2, 3))
    assert theq(theano_code(S.Half), tt.true_div(1, 2))

def test_Integers():
    assert theano_code(sympy.Integer(3)) == 3

def test_factorial():
    n = sympy.Symbol('n')
    assert theano_code(sympy.factorial(n))

def test_theano_function():
    import numpy as np
    f = theano_function([x, y], [x+y], dtypes={x: 'float64', y: 'float64'},
                                     dim=1)
    xx = np.arange(3).astype('float64')
    yy = 2*np.arange(3).astype('float64')
    assert np.linalg.norm(f(xx, yy) - 3*np.arange(3)) < 1e-9

<commit_msg>test theano_function<commit_after>from sympy.external import import_module

theano = import_module('theano')
if theano:
    ts = theano.scalar
    tt = theano.tensor
    xt, yt, zt = [tt.scalar(name, 'floatX') for name in 'xyz']
else:
    #bin/test will not execute any tests now
    disabled = True

import sympy
from sympy import S
sy = sympy
from sympy.abc import x, y, z, a, b, c
from sympy.printing.theanocode import (theano_code, dim_handling,
        theano_function)

def theq(a, b):
    """ theano equality """
    astr = theano.printing.debugprint(a, file='str')
    bstr = theano.printing.debugprint(b, file='str')

    if not astr == bstr:
        print
        print astr
        print bstr

    return astr == bstr

def test_symbol():
    xt = theano_code(x)
    assert isinstance(xt, (tt.TensorVariable, ts.ScalarVariable))
    assert xt.name == x.name

    assert theano_code(x, broadcastables={x: (False,)}).broadcastable == (False,)
    assert theano_code(x, broadcastables={x: (False,)}).name == x.name

def test_add():
    expr = x + y
    comp = theano_code(expr)
    assert comp.owner.op == theano.tensor.add

    comp = theano_code(expr, broadcastables={x: (False,), y: (False,)})
    assert comp.broadcastable == (False,)

    comp = theano_code(expr, broadcastables={x: (False, True), y: (False, False)})
    assert comp.broadcastable == (False, False)

def test_trig():
    assert theq(theano_code(sympy.sin(x)), tt.sin(xt))
    assert theq(theano_code(sympy.tan(x)), tt.tan(xt))

def test_many():
    expr = sy.exp(x**2 + sy.cos(y)) * sy.log(2*z)
    comp = theano_code(expr)
    expected = tt.exp(xt**2 + tt.cos(yt)) * tt.log(2*zt)
    # assert theq(comp, expected)

def test_dtype():
    assert theano_code(x, dtypes={x: 'float32'}).type.dtype == 'float32'
    assert theano_code(x, dtypes={x: 'float64'}).type.dtype == 'float64'
    assert theano_code(x+1, dtypes={x: 'float32'}).type.dtype == 'float32'
    assert theano_code(x+y, dtypes={x: 'float64', y: 'float32'}).type.dtype == 'float64'

def test_MatrixSymbol():
    X = sympy.MatrixSymbol('X', 4, 5)
    Xt = theano_code(X)
    assert isinstance(Xt, tt.TensorVariable)
    assert Xt.broadcastable == (False, False)

def test_MatMul():
    X = sympy.MatrixSymbol('X', 4, 4)
    Y = sympy.MatrixSymbol('X', 4, 4)
    Z = sympy.MatrixSymbol('X', 4, 4)
    expr = X*Y*Z
    assert isinstance(theano_code(expr).owner.op, tt.Dot)

def test_Transpose():
    X = sympy.MatrixSymbol('X', 4, 4)
    assert isinstance(theano_code(X.T).owner.op, tt.DimShuffle)

def test_MatAdd():
    X = sympy.MatrixSymbol('X', 4, 4)
    Y = sympy.MatrixSymbol('X', 4, 4)
    Z = sympy.MatrixSymbol('X', 4, 4)
    expr = X+Y+Z
    assert isinstance(theano_code(expr).owner.op, tt.Elemwise)

def test_symbols_are_created_once():
    expr = x**x
    comp = theano_code(expr)

    assert theq(comp, xt**xt)

def test_dim_handling():
    assert dim_handling([x], dim=2) == {x: (False, False)}
    assert dim_handling([x, y], dims={x: 1, y: 2}) == {x: (False, True),
                                                       y: (False, False)}
    assert dim_handling([x], broadcastables={x: (False,)}) == {x: (False,)}

def test_Rationals():
    assert theq(theano_code(sympy.Integer(2) / 3), tt.true_div(2, 3))
    assert theq(theano_code(S.Half), tt.true_div(1, 2))

def test_Integers():
    assert theano_code(sympy.Integer(3)) == 3

def test_factorial():
    n = sympy.Symbol('n')
    assert theano_code(sympy.factorial(n))

def test_theano_function_simple():
    f = theano_function([x, y], [x+y])
    assert f(2, 3) == 5

def test_theano_function_numpy():
    import numpy as np
    f = theano_function([x, y], [x+y], dim=1)
    assert np.linalg.norm(f([1, 2], [3, 4]) - np.asarray([4, 6])) < 1e-9

    f = theano_function([x, y], [x+y], dtypes={x: 'float64', y: 'float64'},
                                     dim=1)
    xx = np.arange(3).astype('float64')
    yy = 2*np.arange(3).astype('float64')
    assert np.linalg.norm(f(xx, yy) - 3*np.arange(3)) < 1e-9

<|endoftext|>


<commit_before>from myhdl import *

def mux(sel, ls_di, do):
    """ Multiplexes a list of input signals to an output signal
            do = sl_di[sel]

            sel - select index
            ls_di - list of input signals
            do - output signals

    """
    N = len(ls_di)
    @always_comb
    def _mux():
        do.next = 0
        for i in range(N):
            if i == sel:
                do.next = ls_di[i]
    return _mux

def demux(sel, di, ls_do):
    """ Demultiplexes an input signal to a list of output signals
            ls_do[sel] =  di

            sel - select index
            di - input signal
            ls_do - list of output signals
    """
    N = len(ls_do)
    @always_comb
    def _demux():
        for i in range(N):
            ls_do[i].next = 0
            if i == sel:
                ls_do[i].next = di
    return _demux

def ls_mux(sel, lsls_di, ls_do):
    """ Multiplexes a list of input signal structures to an output structure.
        A structure is represented by a list of signals: [signal_1, signal_2, ..., signal_n]
            ls_do[0] = lsls_di[sel][0]
            ls_do[1] = lsls_di[sel][1]
            ...
            ls_do[n] = lsls_di[sel][n]

            sel - select index
            lsls_di - list of input signal structures: [[sig, sig, ..., sig], [sig, sig, ..., sig], ..., [sig, sig, ..., sig]]
            ls_do - output signal structure: [sig, sig, ..., sig]
    """
    N = len(ls_do)
    lsls_in = zip(*lsls_di)
    return [mux(sel, lsls_in[i], ls_do[i]) for i in range(N)]

def ls_demux(sel, ls_di, lsls_do):
    """ Demultiplexes an input signal structure to list of output structures.
        A structure is represented by a list of signals: [signal_1, signal_2, ..., signal_n]
            lsls_do[sel][0] = ls_di[0]
            lsls_do[sel][1] = ls_di[1]
            ...
            lsls_do[sel][n] = ls_di[n]

            sel - select index
            ls_di - input signal structure: [sig, sig, ..., sig]
            lsls_do - list of output signal structures: [[sig, sig, ..., sig], [sig, sig, ..., sig], ..., [sig, sig, ..., sig]]
    """
    N = len (ls_di)
    lsls_out = zip(*lsls_do)
    return [demux(sel, ls_di[i], lsls_out[i])for i in range(N)]

if __name__ == '__main__':
    pass<commit_msg>ls_mux: converted the output of zip() from tuple to list<commit_after>from myhdl import *

def mux(sel, ls_di, do):
    """ Multiplexes a list of input signals to an output signal
            do = sl_di[sel]

            sel - select index
            ls_di - list of input signals
            do - output signals

    """
    N = len(ls_di)
    @always_comb
    def _mux():
        do.next = 0
        for i in range(N):
            if i == sel:
                do.next = ls_di[i]
    return _mux

def demux(sel, di, ls_do):
    """ Demultiplexes an input signal to a list of output signals
            ls_do[sel] =  di

            sel - select index
            di - input signal
            ls_do - list of output signals
    """
    N = len(ls_do)
    @always_comb
    def _demux():
        for i in range(N):
            ls_do[i].next = 0
            if i == sel:
                ls_do[i].next = di
    return _demux

def ls_mux(sel, lsls_di, ls_do):
    """ Multiplexes a list of input signal structures to an output structure.
        A structure is represented by a list of signals: [signal_1, signal_2, ..., signal_n]
            ls_do[0] = lsls_di[sel][0]
            ls_do[1] = lsls_di[sel][1]
            ...
            ls_do[n] = lsls_di[sel][n]

            sel - select index
            lsls_di - list of input signal structures: [[sig, sig, ..., sig], [sig, sig, ..., sig], ..., [sig, sig, ..., sig]]
            ls_do - output signal structure: [sig, sig, ..., sig]
    """
    N = len(ls_do)
    lsls_in = [list(x) for x in zip(*lsls_di)]
    _mux =[mux(sel, lsls_in[i], ls_do[i]) for i in range(N)]
    return instances()

def ls_demux(sel, ls_di, lsls_do):
    """ Demultiplexes an input signal structure to list of output structures.
        A structure is represented by a list of signals: [signal_1, signal_2, ..., signal_n]
            lsls_do[sel][0] = ls_di[0]
            lsls_do[sel][1] = ls_di[1]
            ...
            lsls_do[sel][n] = ls_di[n]

            sel - select index
            ls_di - input signal structure: [sig, sig, ..., sig]
            lsls_do - list of output signal structures: [[sig, sig, ..., sig], [sig, sig, ..., sig], ..., [sig, sig, ..., sig]]
    """
    N = len (ls_di)
    lsls_out = zip(*lsls_do)
    return [demux(sel, ls_di[i], lsls_out[i])for i in range(N)]

if __name__ == '__main__':
    pass<|endoftext|>


<commit_before>import json
from urllib import urlopen

class Client:
    """A very simple client for accessing the regulation and meta data."""
    _reg_cache = {}

    def _dfs_search(self, reg_tree, index):
        """Find the matching node in the tree (if it exists)"""
        if reg_tree['label']['text'] == index:
            return reg_tree
        for child in reg_tree['children']:
            child_search = self._dfs_search(child, index)
            if child_search:
                return child_search

    def _use_reg_cache(self, label, version):
        """See if we've already grabbed that id. Cache results if not.
        @todo: add a timeout"""
        if (label, version) in Client._reg_cache:
            return Client._reg_cache[(label, version)]
        for cache_label, cache_version in Client._reg_cache:
            if cache_version == version and label.startswith(cache_label):
                matching_child = self._dfs_search(
                    Client._reg_cache[(cache_label, cache_version)], label)
                if matching_child:
                    Client._reg_cache[(label, version)] = matching_child
                    return matching_child
        Client._reg_cache[(label, version)] = self._get(
            "regulation/%s/%s" % (label, version))
        return Client._reg_cache[(label, version)]

    def __init__(self, base_url):
        self.base_url = base_url

    def regulation(self, label, version):
        """End point for regulation JSON. Return the result as a dict"""
        return self._use_reg_cache(label, version)

    def layer(self, layer_name, label, version):
        """End point for layer JSON. Return the result as a list"""
        return self._get("layer/%s/%s/%s" % (layer_name, label, version))

    def notices(self):
        """End point for notice searching. Right now, just a list"""
        return self._get("notice")

    def notice(self, document_number):
        """End point for retrieving a single notice."""
        return self._get("notice/%s" % document_number)

    def _get(self, suffix):
        """Actually make the GET request. Assume the result is JSON. Right
        now, there is no error handling"""
        try:
            result = urlopen(self.base_url + suffix)
        except IOError: #   Hack to deal with local FS vs API
            result = urlopen(self.base_url + suffix + "/index.html")
        content = result.read()
        result.close()
        return json.loads(content)
<commit_msg>The HTMLBuilder mutates the regulation tree it is given. To prevent this, return a deep copy from cache<commit_after>import copy
import json
from urllib import urlopen

class Client:
    """A very simple client for accessing the regulation and meta data."""
    _reg_cache = {}

    def _dfs_search(self, reg_tree, index):
        """Find the matching node in the tree (if it exists)"""
        if reg_tree['label']['text'] == index:
            return reg_tree
        for child in reg_tree['children']:
            child_search = self._dfs_search(child, index)
            if child_search:
                return child_search

    def _use_reg_cache(self, label, version):
        """See if we've already grabbed that id. Cache results if not.
        @todo: add a timeout"""
        if (label, version) in Client._reg_cache:
            return Client._reg_cache[(label, version)]
        for cache_label, cache_version in Client._reg_cache:
            if cache_version == version and label.startswith(cache_label):
                matching_child = self._dfs_search(
                    Client._reg_cache[(cache_label, cache_version)], label)
                if matching_child:
                    Client._reg_cache[(label, version)] = matching_child
                    return matching_child
        Client._reg_cache[(label, version)] = self._get(
            "regulation/%s/%s" % (label, version))
        return Client._reg_cache[(label, version)]

    def __init__(self, base_url):
        self.base_url = base_url

    def regulation(self, label, version):
        """End point for regulation JSON. Return the result as a dict"""
        return copy.deepcopy(self._use_reg_cache(label, version))
        #return self._get("regulation/%s/%s" % (label, version))

    def layer(self, layer_name, label, version):
        """End point for layer JSON. Return the result as a list"""
        return self._get("layer/%s/%s/%s" % (layer_name, label, version))

    def notices(self):
        """End point for notice searching. Right now, just a list"""
        return self._get("notice")

    def notice(self, document_number):
        """End point for retrieving a single notice."""
        return self._get("notice/%s" % document_number)

    def _get(self, suffix):
        """Actually make the GET request. Assume the result is JSON. Right
        now, there is no error handling"""
        try:
            result = urlopen(self.base_url + suffix)
        except IOError: #   Hack to deal with local FS vs API
            result = urlopen(self.base_url + suffix + "/index.html")
        content = result.read()
        result.close()
        return json.loads(content)
<|endoftext|>


<commit_before>from spyvm import model_display, model
from spyvm.error import PrimitiveFailedError, MetaPrimFailed
from spyvm.storage import AbstractCachingShadow
from spyvm.plugins.plugin import Plugin

from rpython.rlib import jit, objectmodel
from rpython.rlib.rarithmetic import r_uint, intmask

class SimulationPluginClass(Plugin):
    def simulate(self, w_name, name, interp, s_frame, argcount, w_method):
        if not interp.image.w_simulatePrimitive:
            raise PrimitiveFailedError("No Simulation found")

        from spyvm.interpreter import Return
        from spyvm.error import MethodNotFound

        w_arguments = s_frame.peek_n(argcount)
        w_rcvr = s_frame.peek(argcount)

        s_class = w_rcvr.class_shadow(interp.space)
        try:
            s_class.lookup(interp.image.w_simulatePrimitive)
        except MethodNotFound:
            raise PrimitiveFailedError("%s doesn't implement %s" % (s_class.getname(), "simulatePrimitive:args:"))

        s_frame.push(w_rcvr)
        s_frame.push(w_name)
        s_frame.push(interp.space.wrap_list_unroll_safe(w_arguments))

        s_fallback = w_method.create_frame(interp.space, w_rcvr, w_arguments)
        s_fallback._s_sender = s_frame

        try:
            s_frame._sendSelector(interp.image.w_simulatePrimitive, 2, interp, w_rcvr, w_rcvr.class_shadow(interp.space), s_fallback=s_fallback)
        except Return, ret:
            # must clean the stack, including the rcvr
            s_frame.pop_n(argcount + 1)
            s_frame.push(ret.value)
            return True
        except MetaPrimFailed, e:
            interp.stack_frame(s_fallback, None)
            return False
        else:
            raise PrimitiveFailedError

SimulationPlugin = SimulationPluginClass()
<commit_msg>fixed pypy-error and documented simulation plugin<commit_after>from spyvm import model_display, model
from spyvm.error import PrimitiveFailedError, MetaPrimFailed
from spyvm.storage import AbstractCachingShadow
from spyvm.plugins.plugin import Plugin

from rpython.rlib import jit, objectmodel
from rpython.rlib.rarithmetic import r_uint, intmask

# If an EXTERNAL_CALL for the given moduleName and functionName is not found,
# the SimulationPlugin is used to enable the image simulating that primitive.
# For that, the simulatePrimitive: aFunctionName args: aCollection message is sent
# to the receiver of the EXTERNAL_CALL primitive.
# Note that the moduleName parameter of EXTERNAL_CALL is omitted and has no influence on the simulation!

# simulatePrimitive:args: may return a value to pass it to the primitive caller and
# should raise primitive 255 META_PRIM_FAILED (e.g. via InterpreterProxy>>primitiveFailFor:)
# if the primitive failed and the fallback code should be run.

# Example:
# BitBlt>>simulatePrimitive: aFunctionName args: aCollection
#     aFunctionName = 'primitiveCopyBits' ifTrue: [ ^self copyBitsSimulated ].
#     aFunctionName = 'primitiveWarpBits' ifTrue: [ ^self warpBitsSimulated: (args at: 1) sourceMap: (args at: 2) ].
#     ^InterpreterProxy new primitiveFailFor: 255.

class SimulationPluginClass(Plugin):
    def simulate(self, w_name, name, interp, s_frame, argcount, w_method):
        if not interp.image.w_simulatePrimitive:
            raise PrimitiveFailedError("No Simulation found")

        from spyvm.interpreter import Return
        from spyvm.error import MethodNotFound

        w_arguments = s_frame.peek_n(argcount)
        w_rcvr = s_frame.peek(argcount)

        s_class = w_rcvr.class_shadow(interp.space)
        try:
            s_class.lookup(interp.image.w_simulatePrimitive)
        except MethodNotFound:
            raise PrimitiveFailedError("%s doesn't implement %s" % (s_class.getname(), "simulatePrimitive:args:"))

        s_frame.push(w_rcvr)
        s_frame.push(w_name)
        s_frame.push(interp.space.wrap_list_unroll_safe(w_arguments))

        s_fallback = w_method.create_frame(interp.space, w_rcvr, w_arguments)
        s_fallback._s_sender = s_frame

        try:
            s_frame._sendSelector(interp.image.w_simulatePrimitive, 2, interp, w_rcvr, w_rcvr.class_shadow(interp.space), s_fallback=s_fallback)
        except Return, ret:
            # must clean the stack, including the rcvr
            s_frame.pop_n(argcount + 1)
            s_frame.push(ret.value)
            return w_rcvr
        except MetaPrimFailed, e:
            interp.stack_frame(s_fallback, None)
            return w_rcvr
        else:
            raise PrimitiveFailedError

SimulationPlugin = SimulationPluginClass()
<|endoftext|>


<commit_before># encoding: utf-8

from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "0.16.2"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<commit_msg>Modify __copyright__<commit_after># encoding: utf-8

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016, {}".format(__author__)
__license__ = "MIT License"
__version__ = "0.16.2"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
dbcommand

Send and execute a command in database. Optionally get a result back. See usage.

NB! Depends on dboperator which takes care of connection and other db stuff.
"""
import sys,getopt
from time import localtime, strftime

import dboperator

def show(message):
  print strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message

def load(command,expect,verbose=False):
  if verbose: show("begin with "+command) # no point in trying to show expect!

  sql = command
  ret = None
  try:
    # quick hack for getting return value. do fix me!
    if expect:
      resql = dboperator.get(sql)
      if verbose: show("command returned %d rows"%(len(resql)))
      # print out (as a return value) the entire result or row count or what, some kind of figuring?
      if expect=='row count':
        ret = len(resql)
      elif len(resql)==1 and expect in resql[0]:
        ret = resql[0][expect]
      else:
        ret = resql
    else:
      dboperator.execute(sql)
  except:
    show("Something went wrong. Over and out.")
    dboperator.close()
    exit(2) # lopeta virheeseen

  if verbose: show("ready")
  return ret

def usage():
  print """
usage: dbcommand.py -c|--command <string> [-e|--expect <string>] [-v|--verbose] [-r|--return]

command is mandatory argument. The SQL to execute.
expect  is for getting results from a query.
        For ease of use one can tell from one row return query the column which value is expected.
        Special case is "row count" which will just return the number of rows query returns.
        Somewhat default behaviour is to return the whole result set as an array of dicts.
        You can achieve default behaviour by giving a value to this argument that is not part
        of column names in result, for ex. "*", and/or results give exactly NOT one row.
return is an optional argument. By default the dbcommand prints the result to shell. With this
        paremeter the dbcommand returns the result to user.
"""

def main(argv):
  # variables that are given as arguments with possible default values
  command = ""
  expect = ""
  verbose = False
  return_wanted = False

  try:
    opts, args = getopt.getopt(argv,"c:e:v:r",["command=","expect=","verbose","return"])
  except getopt.GetoptError as err:
    print(err)
    usage()
    sys.exit(2)
  for opt, arg in opts:
    if opt in ("-c", "--command"): command = arg
    elif opt in ("-e", "--expect"): expect = arg
    elif opt in ("-v", "--verbose"): verbose = True
    elif opt in ("-r", "--return"): return_wanted = True
  if not command:
    usage()
    sys.exit(2)

  if expect:
    verbose = False # would interfere with "return" value
  ret = load(command,expect,verbose)
  if expect:
     if return_wanted:
         return ret
     else:
         print ret

  #dboperator.close()  this to be optional ?

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>adding debug-info to failed dboperator<commit_after>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
dbcommand

Send and execute a command in database. Optionally get a result back. See usage.

NB! Depends on dboperator which takes care of connection and other db stuff.
"""
import sys,getopt
from time import localtime, strftime

import dboperator

def show(message):
  print strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message

def load(command,expect,verbose=False):
  if verbose: show("begin with "+command) # no point in trying to show expect!

  sql = command
  ret = None
  try:
    # quick hack for getting return value. do fix me!
    if expect:
      resql = dboperator.get(sql)
      if verbose: show("command returned %d rows"%(len(resql)))
      # print out (as a return value) the entire result or row count or what, some kind of figuring?
      if expect=='row count':
        ret = len(resql)
      elif len(resql)==1 and expect in resql[0]:
        ret = resql[0][expect]
      else:
        ret = resql
    else:
      dboperator.execute(sql)
  except Exception as e:
    print(e)
    show("Something went wrong. Over and out.")
    dboperator.close() this to be optional ?
    exit(2) # lopeta virheeseen

  if verbose: show("ready")
  return ret

def usage():
  print """
usage: dbcommand.py -c|--command <string> [-e|--expect <string>] [-v|--verbose] [-r|--return]

command is mandatory argument. The SQL to execute.
expect  is for getting results from a query.
        For ease of use one can tell from one row return query the column which value is expected.
        Special case is "row count" which will just return the number of rows query returns.
        Somewhat default behaviour is to return the whole result set as an array of dicts.
        You can achieve default behaviour by giving a value to this argument that is not part
        of column names in result, for ex. "*", and/or results give exactly NOT one row.
return is an optional argument. By default the dbcommand prints the result to shell. With this
        paremeter the dbcommand returns the result to user.
"""

def main(argv):
  # variables that are given as arguments with possible default values
  command = ""
  expect = ""
  verbose = False
  return_wanted = False

  try:
    opts, args = getopt.getopt(argv,"c:e:v:r",["command=","expect=","verbose","return"])
  except getopt.GetoptError as err:
    print(err)
    usage()
    sys.exit(2)
  for opt, arg in opts:
    if opt in ("-c", "--command"): command = arg
    elif opt in ("-e", "--expect"): expect = arg
    elif opt in ("-v", "--verbose"): verbose = True
    elif opt in ("-r", "--return"): return_wanted = True
  if not command:
    usage()
    sys.exit(2)

  if expect:
    verbose = False # would interfere with "return" value
  ret = load(command,expect,verbose)
  if expect:
     if return_wanted:
         return ret
     else:
         print ret

  #dboperator.close()  this to be optional ?

if __name__ == "__main__":
  main(sys.argv[1:])
<|endoftext|>


<commit_before># Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from contracts import contract
from neat.contracts_extra import *

from sqlalchemy import *

from neat.db import Database

import logging
log = logging.getLogger(__name__)

@contract
def init_db(sql_connection):
    """ Initialize the database.

    :param sql_connection: A database connection URL.
     :type sql_connection: str

    :return: The initialized database.
     :rtype: Database
    """
    engine = create_engine(sql_connection)  # 'sqlite:///:memory:'
    metadata = MetaData()
    metadata.bind = engine

    hosts = Table('hosts', metadata,
                  Column('id', Integer, primary_key=True),
                  Column('hostname', String(255), nullable=False),
                  Column('cpu_mhz', Integer, nullable=False),
                  Column('ram', Integer, nullable=False))

    vms = Table('vms', metadata,
                Column('id', Integer, primary_key=True),
                Column('uuid', String(36), nullable=False))

    vm_resource_usage = \
        Table('vm_resource_usage', metadata,
              Column('id', Integer, primary_key=True),
              Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False),
              Column('timestamp', DateTime,
                     server_default=text('CURRENT_TIMESTAMP')),
              Column('cpu_mhz', Integer, nullable=False))

    metadata.create_all()
    connection = engine.connect()
    db = Database(connection, hosts, vms, vm_resource_usage)

    log.info('Initialized a DB connection to %s', sql_connection)
    return db
<commit_msg>Fixed the default value of the timestamp<commit_after># Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from contracts import contract
from neat.contracts_extra import *

from sqlalchemy import *
from sqlalchemy.sql import func

from neat.db import Database

import logging
log = logging.getLogger(__name__)

@contract
def init_db(sql_connection):
    """ Initialize the database.

    :param sql_connection: A database connection URL.
     :type sql_connection: str

    :return: The initialized database.
     :rtype: Database
    """
    engine = create_engine(sql_connection)  # 'sqlite:///:memory:'
    metadata = MetaData()
    metadata.bind = engine

    hosts = Table('hosts', metadata,
                  Column('id', Integer, primary_key=True),
                  Column('hostname', String(255), nullable=False),
                  Column('cpu_mhz', Integer, nullable=False),
                  Column('ram', Integer, nullable=False))

    vms = Table('vms', metadata,
                Column('id', Integer, primary_key=True),
                Column('uuid', String(36), nullable=False))

    vm_resource_usage = \
        Table('vm_resource_usage', metadata,
              Column('id', Integer, primary_key=True),
              Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False),
              Column('timestamp', DateTime, default=func.now()),
                     #server_default=text('CURRENT_TIMESTAMP')),
              Column('cpu_mhz', Integer, nullable=False))

    metadata.create_all()
    connection = engine.connect()
    db = Database(connection, hosts, vms, vm_resource_usage)

    log.info('Initialized a DB connection to %s', sql_connection)
    return db
<|endoftext|>


<commit_before>from SimPEG import *
import matplotlib.pyplot as plt

class LinearSurvey(Survey.BaseSurvey):
    def projectFields(self, u):
        return u

class LinearProblem(Problem.BaseProblem):
    """docstring for LinearProblem"""

    surveyPair = LinearSurvey

    def __init__(self, model, G, **kwargs):
        Problem.BaseProblem.__init__(self, model, **kwargs)
        self.G = G

    def fields(self, m, u=None):
        return self.G.dot(m)

    def Jvec(self, m, v, u=None):
        return self.G.dot(v)

    def Jtvec(self, m, v, u=None):
        return self.G.T.dot(v)

def example(N):
    M = Mesh.TensorMesh([N])

    nk = 20
    jk = np.linspace(1.,20.,nk)
    p = -0.25
    q = 0.25

    g = lambda k: np.exp(p*jk[k]*M.vectorCCx)*np.cos(2*np.pi*q*jk[k]*M.vectorCCx)

    G = np.empty((nk, M.nC))

    for i in range(nk):
        G[i,:] = g(i)

    mtrue = np.zeros(M.nC)
    mtrue[M.vectorCCx > 0.3] = 1.
    mtrue[M.vectorCCx > 0.45] = -0.5
    mtrue[M.vectorCCx > 0.6] = 0

    model = Model.BaseModel(M)
    prob = LinearProblem(model, G)
    survey = prob.createSyntheticSurvey(mtrue, std=0.01)

    return prob, survey, model

if __name__ == '__main__':

    prob, survey, model = example(100)
    M = prob.mesh

    reg = Regularization.Tikhonov(model)
    objFunc = ObjFunction.BaseObjFunction(survey, reg)
    opt = Optimization.InexactGaussNewton(maxIter=20)
    inv = Inversion.BaseInversion(objFunc, opt)
    m0 = np.zeros_like(survey.mtrue)

    mrec = inv.run(m0)

    plt.figure(1)
    for i in range(prob.G.shape[0]):
        plt.plot(prob.G[i,:])

    plt.figure(2)

    plt.plot(M.vectorCCx, survey.mtrue, 'b-')
    plt.plot(M.vectorCCx, mrec, 'r-')

    plt.show()
<commit_msg>Minor updates to Linear Problem.<commit_after>from SimPEG import *

class LinearSurvey(Survey.BaseSurvey):
    def projectFields(self, u):
        return u

class LinearProblem(Problem.BaseProblem):
    """docstring for LinearProblem"""

    surveyPair = LinearSurvey

    def __init__(self, model, G, **kwargs):
        Problem.BaseProblem.__init__(self, model, **kwargs)
        self.G = G

    def fields(self, m, u=None):
        return self.G.dot(m)

    def Jvec(self, m, v, u=None):
        return self.G.dot(v)

    def Jtvec(self, m, v, u=None):
        return self.G.T.dot(v)

def example(N):
    M = Mesh.TensorMesh([N])

    nk = 20
    jk = np.linspace(1.,20.,nk)
    p = -0.25
    q = 0.25

    g = lambda k: np.exp(p*jk[k]*M.vectorCCx)*np.cos(2*np.pi*q*jk[k]*M.vectorCCx)

    G = np.empty((nk, M.nC))

    for i in range(nk):
        G[i,:] = g(i)

    mtrue = np.zeros(M.nC)
    mtrue[M.vectorCCx > 0.3] = 1.
    mtrue[M.vectorCCx > 0.45] = -0.5
    mtrue[M.vectorCCx > 0.6] = 0

    model = Model.BaseModel(M)
    prob = LinearProblem(model, G)
    survey = prob.createSyntheticSurvey(mtrue, std=0.01)

    return prob, survey, model

if __name__ == '__main__':

    import matplotlib.pyplot as plt

    prob, survey, model = example(100)
    M = prob.mesh

    reg = Regularization.Tikhonov(model)
    beta = Parameters.BetaSchedule()
    objFunc = ObjFunction.BaseObjFunction(survey, reg, beta=beta)
    opt = Optimization.InexactGaussNewton(maxIter=20)
    inv = Inversion.BaseInversion(objFunc, opt)
    m0 = np.zeros_like(survey.mtrue)

    mrec = inv.run(m0)

    plt.figure(1)
    for i in range(prob.G.shape[0]):
        plt.plot(prob.G[i,:])

    plt.figure(2)

    plt.plot(M.vectorCCx, survey.mtrue, 'b-')
    plt.plot(M.vectorCCx, mrec, 'r-')

    plt.show()
<|endoftext|>


<commit_before># (C) Copyright 2017 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import glob
import logging
import os
import subprocess
import re

import yaml

from distutils.version import LooseVersion
from dockerfile_parse import DockerfileParser

from dbuild.tag import (TAG_REGEXES, DockerTag,
                        parse_docker_tag, docker_tags_from_args)
from dbuild.verb import Argument, VerbException

REGEX_MODULE = re.compile(r'^[a-z0-9\-]+$')
REGEX_DOCKERFILE_REBUILD = re.compile(r'^REBUILD_([A-Z_]+)=.+$')

ARG_TAG = Argument('tag', TAG_REGEXES)

ARG_BUILD_ARG = Argument('build_arg', re.compile(r'^([\w_]+)=(.*)$'))
ARG_VARIANT = Argument('variant', re.compile(r'^(\w[\w_.-]*)$'))
ARG_REBUILD = Argument('rebuild', re.compile(r'^@(\w[\w_.-]*)$'))
ARG_APPEND = Argument('append', re.compile(r'^\+$'))

MIN_DOCKER_VERSION = LooseVersion('1.13.0')

logger = logging.getLogger(__name__)
config_cache = {}
dockerfile_cache = {}

def load_config(base_path, module):
    conf_path = os.path.join(base_path, module, 'build.yml')
    if conf_path in config_cache:
        return config_cache[conf_path]

    if not os.path.exists(conf_path):
        return {}

    with open(conf_path, 'r') as f:
        ret = yaml.safe_load(f)
        config_cache[conf_path] = ret
        return ret

def get_canonical_variants(config, variants):
    canonical_variants = set()
    for variant in variants:
        if variant == 'all':
            canonical_variants.update(map(lambda v: v['tag'],
                                          config['variants']))
            continue

        found = False
        for defined_variant in config['variants']:
            if variant == defined_variant['tag']:
                canonical_variants.add(variant)
                found = True
            elif variant in defined_variant['aliases']:
                canonical_variants.add(defined_variant['tag'])
                found = True

        if not found:
            # TODO maybe just ignore this?
            # then many variants could be built across modules that might not
            # share names
            logger.error('Variant not found in build.yml for %s: %s. '
                         'Note that the same variants must exist in all '
                         'modules.', config['repository'], variant)
            raise VerbException()

    return canonical_variants

def get_variant(config, variant_tag):
    if 'variants' not in config:
        return {}

    for variant in config['variants']:
        if variant['tag'] == variant_tag:
            return variant

    return None

def resolve_variants(verb_args, config, check_tag=True):
    base_tag = DockerTag().mutate(repository=config.get('repository', None))

    variants = []
    for arg in filter(lambda a: a.type == 'variant', verb_args):
        variants.append(arg.groups[0])

    if variants and 'variants' not in config:
        logger.error('No variants defined in build.yml!')
        raise VerbException()

    tag_args = filter(lambda a: a.type == 'tag', verb_args)
    append = filter(lambda a: a.type == 'append', verb_args)

    canonical_variants = get_canonical_variants(config, variants)
    if canonical_variants:
        resolved_variants = []
        for variant_tag in canonical_variants:
            variant = get_variant(config, variant_tag)

            variant_base_tag = base_tag.mutate(tag=variant_tag)
            if 'repository' in variant:
                variant_base_tag = variant_base_tag.mutate(
                    repository=variant['repository'])

            tags = []
            if variant_base_tag.is_complete():
                tags.append(variant_base_tag)

            if 'aliases' in variant:
                tags.extend(docker_tags_from_args(variant['aliases'],
                                                  variant_base_tag))

            if tag_args:
                str_args = [t.value for t in tag_args]
                dtags = docker_tags_from_args(str_args, variant_base_tag,
                                              check_tag)
                if append:
                    tags.extend(dtags)
                else:
                    tags = dtags

            if not tags:
                raise VerbException(
                    'At least one complete tag is required. '
                    'Base: %r, known: %r' % (variant_base_tag, tags))

            resolved_variants.append({
                'variant_tag': variant_tag,
                'tags': tags
            })

        return resolved_variants
    else:
        logger.info('No variant given, using inferred base tag: %r', base_tag)
        str_args = [t.value for t in tag_args]
        tags = docker_tags_from_args(str_args, base_tag, check_tag)
        if not tags:
            raise VerbException('No valid Docker tag given in arguments, '
                                'known: %r', tags)

        return [{
            'variant_tag': None,
            'tags': tags
        }]

def load_dockerfile(base_path, module):
    dockerfile_path = os.path.join(base_path, module, 'Dockerfile')
    if dockerfile_path in dockerfile_cache:
        return dockerfile_cache[dockerfile_path]

    with open(dockerfile_path, 'r') as f:
        p = DockerfileParser()
        p.content = f.read()
        dockerfile_cache[dockerfile_path] = p
        return p

def get_rebuild_targets(dockerfile):
    targets = []
    for ins in dockerfile.structure:
        if ins['instruction'] != 'ARG':
            continue

        m = REGEX_DOCKERFILE_REBUILD.match(ins['value'])
        if m:
            targets.append(m.group(1).lower())

    return targets

def list_modules(path):
    all_modules = map(lambda p: os.path.basename(os.path.dirname(p)),
                      glob.glob(os.path.join(path, '*/Dockerfile')))

    valid_modules = []
    for module in all_modules:
        m = REGEX_MODULE.match(module)
        if m:
            valid_modules.append(module)
        else:
            logging.debug('Ignoring module with invalid name: %s', module)

    return valid_modules

class SubprocessException(Exception):
    def __init__(self, retcode, stdout, stderr):
        super(SubprocessException, self).__init__(stderr)

        self.retcode = retcode
        self.stdout = stdout
        self.stderr = stderr

    def __str__(self):
        return 'CaptureException(retcode=%s, stdout=%r, stderr=%r)' % (
            self.retcode,
            self.stdout,
            self.stderr
        )

def exec_docker(args):
    logger.debug('Running docker: %r', args)
    p = subprocess.Popen(['docker'] + args)

    ret = p.wait()
    if p.returncode != 0:
        raise SubprocessException(ret, None, None)

def capture_docker(args):
    logger.debug('Capturing docker: %r', args)
    p = subprocess.Popen(['docker'] + args,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

    out, err = p.communicate()
    if p.returncode != 0:
        raise SubprocessException(p.returncode, out, err)

    return out, err

class InvalidDockerVersionException(Exception):
    pass

def get_docker_client_version():
    out, err = capture_docker(['version', '-f', '{{.Client.Version}}'])
    return LooseVersion(out.strip())

def verify_docker_version():
    docker_client_version = get_docker_client_version()
    if docker_client_version >= MIN_DOCKER_VERSION:
        logger.debug('Docker version %s meets requirement >= %s',
                     docker_client_version, MIN_DOCKER_VERSION)
    else:
        raise InvalidDockerVersionException(
            'Installed Docker version %s does not meet requirement >= %s' % (
                docker_client_version, MIN_DOCKER_VERSION
            ))
<commit_msg>Add IGNORE_DOCKER_VERSION env var check<commit_after># (C) Copyright 2017 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import glob
import logging
import os
import subprocess
import re

import yaml

from distutils.version import LooseVersion
from dockerfile_parse import DockerfileParser

from dbuild.tag import (TAG_REGEXES, DockerTag,
                        parse_docker_tag, docker_tags_from_args)
from dbuild.verb import Argument, VerbException

REGEX_MODULE = re.compile(r'^[a-z0-9\-]+$')
REGEX_DOCKERFILE_REBUILD = re.compile(r'^REBUILD_([A-Z_]+)=.+$')

ARG_TAG = Argument('tag', TAG_REGEXES)

ARG_BUILD_ARG = Argument('build_arg', re.compile(r'^([\w_]+)=(.*)$'))
ARG_VARIANT = Argument('variant', re.compile(r'^(\w[\w_.-]*)$'))
ARG_REBUILD = Argument('rebuild', re.compile(r'^@(\w[\w_.-]*)$'))
ARG_APPEND = Argument('append', re.compile(r'^\+$'))

MIN_DOCKER_VERSION = LooseVersion('1.13.0')

logger = logging.getLogger(__name__)
config_cache = {}
dockerfile_cache = {}

def load_config(base_path, module):
    conf_path = os.path.join(base_path, module, 'build.yml')
    if conf_path in config_cache:
        return config_cache[conf_path]

    if not os.path.exists(conf_path):
        return {}

    with open(conf_path, 'r') as f:
        ret = yaml.safe_load(f)
        config_cache[conf_path] = ret
        return ret

def get_canonical_variants(config, variants):
    canonical_variants = set()
    for variant in variants:
        if variant == 'all':
            canonical_variants.update(map(lambda v: v['tag'],
                                          config['variants']))
            continue

        found = False
        for defined_variant in config['variants']:
            if variant == defined_variant['tag']:
                canonical_variants.add(variant)
                found = True
            elif variant in defined_variant['aliases']:
                canonical_variants.add(defined_variant['tag'])
                found = True

        if not found:
            # TODO maybe just ignore this?
            # then many variants could be built across modules that might not
            # share names
            logger.error('Variant not found in build.yml for %s: %s. '
                         'Note that the same variants must exist in all '
                         'modules.', config['repository'], variant)
            raise VerbException()

    return canonical_variants

def get_variant(config, variant_tag):
    if 'variants' not in config:
        return {}

    for variant in config['variants']:
        if variant['tag'] == variant_tag:
            return variant

    return None

def resolve_variants(verb_args, config, check_tag=True):
    base_tag = DockerTag().mutate(repository=config.get('repository', None))

    variants = []
    for arg in filter(lambda a: a.type == 'variant', verb_args):
        variants.append(arg.groups[0])

    if variants and 'variants' not in config:
        logger.error('No variants defined in build.yml!')
        raise VerbException()

    tag_args = filter(lambda a: a.type == 'tag', verb_args)
    append = filter(lambda a: a.type == 'append', verb_args)

    canonical_variants = get_canonical_variants(config, variants)
    if canonical_variants:
        resolved_variants = []
        for variant_tag in canonical_variants:
            variant = get_variant(config, variant_tag)

            variant_base_tag = base_tag.mutate(tag=variant_tag)
            if 'repository' in variant:
                variant_base_tag = variant_base_tag.mutate(
                    repository=variant['repository'])

            tags = []
            if variant_base_tag.is_complete():
                tags.append(variant_base_tag)

            if 'aliases' in variant:
                tags.extend(docker_tags_from_args(variant['aliases'],
                                                  variant_base_tag))

            if tag_args:
                str_args = [t.value for t in tag_args]
                dtags = docker_tags_from_args(str_args, variant_base_tag,
                                              check_tag)
                if append:
                    tags.extend(dtags)
                else:
                    tags = dtags

            if not tags:
                raise VerbException(
                    'At least one complete tag is required. '
                    'Base: %r, known: %r' % (variant_base_tag, tags))

            resolved_variants.append({
                'variant_tag': variant_tag,
                'tags': tags
            })

        return resolved_variants
    else:
        logger.info('No variant given, using inferred base tag: %r', base_tag)
        str_args = [t.value for t in tag_args]
        tags = docker_tags_from_args(str_args, base_tag, check_tag)
        if not tags:
            raise VerbException('No valid Docker tag given in arguments, '
                                'known: %r', tags)

        return [{
            'variant_tag': None,
            'tags': tags
        }]

def load_dockerfile(base_path, module):
    dockerfile_path = os.path.join(base_path, module, 'Dockerfile')
    if dockerfile_path in dockerfile_cache:
        return dockerfile_cache[dockerfile_path]

    with open(dockerfile_path, 'r') as f:
        p = DockerfileParser()
        p.content = f.read()
        dockerfile_cache[dockerfile_path] = p
        return p

def get_rebuild_targets(dockerfile):
    targets = []
    for ins in dockerfile.structure:
        if ins['instruction'] != 'ARG':
            continue

        m = REGEX_DOCKERFILE_REBUILD.match(ins['value'])
        if m:
            targets.append(m.group(1).lower())

    return targets

def list_modules(path):
    all_modules = map(lambda p: os.path.basename(os.path.dirname(p)),
                      glob.glob(os.path.join(path, '*/Dockerfile')))

    valid_modules = []
    for module in all_modules:
        m = REGEX_MODULE.match(module)
        if m:
            valid_modules.append(module)
        else:
            logging.debug('Ignoring module with invalid name: %s', module)

    return valid_modules

class SubprocessException(Exception):
    def __init__(self, retcode, stdout, stderr):
        super(SubprocessException, self).__init__(stderr)

        self.retcode = retcode
        self.stdout = stdout
        self.stderr = stderr

    def __str__(self):
        return 'CaptureException(retcode=%s, stdout=%r, stderr=%r)' % (
            self.retcode,
            self.stdout,
            self.stderr
        )

def exec_docker(args):
    logger.debug('Running docker: %r', args)
    p = subprocess.Popen(['docker'] + args)

    ret = p.wait()
    if p.returncode != 0:
        raise SubprocessException(ret, None, None)

def capture_docker(args):
    logger.debug('Capturing docker: %r', args)
    p = subprocess.Popen(['docker'] + args,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

    out, err = p.communicate()
    if p.returncode != 0:
        raise SubprocessException(p.returncode, out, err)

    return out, err

class InvalidDockerVersionException(Exception):
    pass

def get_docker_client_version():
    out, err = capture_docker(['version', '-f', '{{.Client.Version}}'])
    return LooseVersion(out.strip())

def verify_docker_version():
    if os.environ.get('IGNORE_DOCKER_VERSION', 'false') == 'true':
        logger.debug('Skipping Docker version check')
        return

    docker_client_version = get_docker_client_version()
    if docker_client_version >= MIN_DOCKER_VERSION:
        logger.debug('Docker version %s meets requirement >= %s',
                     docker_client_version, MIN_DOCKER_VERSION)
    else:
        raise InvalidDockerVersionException(
            'Installed Docker version %s does not meet requirement >= %s' % (
                docker_client_version, MIN_DOCKER_VERSION
            ))
<|endoftext|>


<commit_before>import operator

from pprint import pprint

from flask import abort, Blueprint, request, Response, url_for, json
from flask.ext.jsonpify import jsonify
#from flask import jsonify
import jsonpatch

from ..extensions import db
from ..models import VoluntaryWork
from ..schemas import VoluntaryWorkListSchema
from ..serializers import (
    MunicipalitySerializer,
    VoluntaryWorkSerializer,
    VoluntaryWorkTypeSerializer,
)
from ..services import (
    VoluntaryWorkEditTokenService,
    VoluntaryWorkEmailConfirmationService
)

voluntary_work = Blueprint(
    name='voluntary_work',
    import_name=__name__,
    url_prefix='/voluntary_works'
)

@voluntary_work.route('')
def index():
    page = request.args.get('page', type=int, default=1)
    pagination = (
        VoluntaryWork.query
        .order_by(db.desc(VoluntaryWork.id))
        .paginate(page=page)
    )
    return jsonify(**_serialize_pagination(pagination))

@voluntary_work.route('/<int:id>')
def get(id):
    voluntary_work = VoluntaryWork.query.filter_by(id=id).one()
    return jsonify(**_serialize([voluntary_work]))

@voluntary_work.route('', methods=['POST'])
def post3():

    schema = VoluntaryWorkListSchema()

    data = schema.deserialize(json.loads(request.data))
    data = data['voluntary_works'][0]
    data.update(data.pop('links'))

    voluntary_work = VoluntaryWork(**data)
    db.session.add(voluntary_work)

    db.session.commit()
#    service = VoluntaryWorkEmailConfirmationService(voluntary_work.id)
#    service.send_confirmation_email()

    token = VoluntaryWorkEditTokenService.get_token(voluntary_work.id)

    ser = _serialize([voluntary_work])
    #ser['voluntary_works'].update({"token": token})

    response = jsonify(**ser)
    response.status_code = 201
    response.location = url_for('.get', id=voluntary_work.id)
    return response

@voluntary_work.route('/create')
def post():

    schema = VoluntaryWorkListSchema()

    try:
        data = json.loads(request.data)
        #data = schema.deserialize(json.loads(request.args['data']))
    except Exception as inst:
        print inst

    data = data['voluntary_works'][0]
    data.update(data.pop('links'))

    voluntary_work = VoluntaryWork(**data)
    db.session.add(voluntary_work)

    db.session.commit()

    service = VoluntaryWorkEmailConfirmationService(voluntary_work.id)
    service.send_confirmation_email()

    token = VoluntaryWorkEditTokenService.get_token(voluntary_work.id)

    ser = _serialize([voluntary_work])
    #ser['voluntary_works']['token'] = token

    response = jsonify(**ser)
    response.status_code = 201
    response.location = url_for('.get', id=voluntary_work.id)
    return response

@voluntary_work.route('/<int:id>', methods=['POST'])
def post2(id):
#    if request.mimetype != 'application/json-patch+json':
#        abort(400)

    print 1

    voluntary_work = VoluntaryWork.query.filter_by(id=id).one()

    data = json.loads(request.data)
    #data = json.loads(request.args['data'])

    edit_token = data['voluntary_works'][0]['token']

    voluntary_work_id = (VoluntaryWorkEditTokenService.get_voluntary_work_id_from_token(edit_token))

    if voluntary_work.id != voluntary_work_id:
        abort(403)

    serializer = VoluntaryWorkSerializer([voluntary_work], many=True)
    json2 = {'voluntary_works': serializer.data}

    try:
        patch = jsonpatch.JsonPatch(data)
    except Exception as es:
        print es

    data = data['voluntary_works'][0]
    data.pop('links')
    data.pop('token')

    try:
        for key, value in data.iteritems():
            setattr(voluntary_work, key, value)
        db.session.commit()
    except Exception as es:
        abort(403)

    #return Response(mimetype='application/json', status=204)
    ser = _serialize([voluntary_work])

    response = jsonify(**ser)
    response.status_code = 200
    response.location = url_for('.get', id=voluntary_work.id)
    return response

def _get_links():
    return {
        'voluntary_works.municipality': {
            'href': (
                url_for('municipality.index', _external=True) +
                '/{voluntary_works.municipality}'
            ),
            'type': 'municipalities'
        },
        'voluntary_works.type': {
            'href': (
                url_for('type.index', _external=True) +
                '/{voluntary_works.type}'
            ),
            'type': 'types'
        }
    }

def _get_linked(voluntary_works):
    relationships = [
        ('type', 'types', 'id', VoluntaryWorkTypeSerializer),
        ('municipality', 'municipalities', 'code', MunicipalitySerializer),
    ]
    linked = {}
    for name, plural_name, primary_key, serializer_cls in relationships:
        items = set(getattr(work, name) for work in voluntary_works)
        items = sorted(items, key=operator.attrgetter(primary_key))
        serializer = serializer_cls(items, many=True)
        linked[plural_name] = serializer.data
    return linked

def _serialize(voluntary_works):
    serializer = VoluntaryWorkSerializer(voluntary_works, many=True)
    return {
        'links': _get_links(),
        'voluntary_works': serializer.data,
        'linked': _get_linked(voluntary_works)
    }

def _serialize_pagination(pagination):
    data = _serialize(pagination.items)
    data['meta'] = {
        'pagination': {
            'page': pagination.page,
            'pages': pagination.pages,
            'per_page': pagination.per_page,
            'total': pagination.total,
        }
    }
    return data
<commit_msg>email, try 3<commit_after>import operator

from pprint import pprint

from flask import abort, Blueprint, request, Response, url_for, json
from flask.ext.jsonpify import jsonify
#from flask import jsonify
import jsonpatch

from ..extensions import db
from ..models import VoluntaryWork
from ..schemas import VoluntaryWorkListSchema
from ..serializers import (
    MunicipalitySerializer,
    VoluntaryWorkSerializer,
    VoluntaryWorkTypeSerializer,
)
from ..services import (
    VoluntaryWorkEditTokenService,
    VoluntaryWorkEmailConfirmationService
)

voluntary_work = Blueprint(
    name='voluntary_work',
    import_name=__name__,
    url_prefix='/voluntary_works'
)

@voluntary_work.route('')
def index():
    page = request.args.get('page', type=int, default=1)
    pagination = (
        VoluntaryWork.query
        .order_by(db.desc(VoluntaryWork.id))
        .paginate(page=page)
    )
    return jsonify(**_serialize_pagination(pagination))

@voluntary_work.route('/<int:id>')
def get(id):
    voluntary_work = VoluntaryWork.query.filter_by(id=id).one()
    return jsonify(**_serialize([voluntary_work]))

@voluntary_work.route('', methods=['POST'])
def post3():

    schema = VoluntaryWorkListSchema()

    data = schema.deserialize(json.loads(request.data))
    data = data['voluntary_works'][0]
    data.update(data.pop('links'))

    voluntary_work = VoluntaryWork(**data)
    db.session.add(voluntary_work)

    db.session.commit()
    service = VoluntaryWorkEmailConfirmationService(voluntary_work.id)
    service.send_confirmation_email()

    token = VoluntaryWorkEditTokenService.get_token(voluntary_work.id)

    ser = _serialize([voluntary_work])
    #ser['voluntary_works'].update({"token": token})

    response = jsonify(**ser)
    response.status_code = 201
    response.location = url_for('.get', id=voluntary_work.id)
    return response

@voluntary_work.route('/create')
def post():

    schema = VoluntaryWorkListSchema()

    try:
        data = json.loads(request.data)
        #data = schema.deserialize(json.loads(request.args['data']))
    except Exception as inst:
        print inst

    data = data['voluntary_works'][0]
    data.update(data.pop('links'))

    voluntary_work = VoluntaryWork(**data)
    db.session.add(voluntary_work)

    db.session.commit()

    service = VoluntaryWorkEmailConfirmationService(voluntary_work.id)
    service.send_confirmation_email()

    token = VoluntaryWorkEditTokenService.get_token(voluntary_work.id)

    ser = _serialize([voluntary_work])
    #ser['voluntary_works']['token'] = token

    response = jsonify(**ser)
    response.status_code = 201
    response.location = url_for('.get', id=voluntary_work.id)
    return response

@voluntary_work.route('/<int:id>', methods=['POST'])
def post2(id):
#    if request.mimetype != 'application/json-patch+json':
#        abort(400)

    print 1

    voluntary_work = VoluntaryWork.query.filter_by(id=id).one()

    data = json.loads(request.data)
    #data = json.loads(request.args['data'])

    edit_token = data['voluntary_works'][0]['token']

    voluntary_work_id = (VoluntaryWorkEditTokenService.get_voluntary_work_id_from_token(edit_token))

    if voluntary_work.id != voluntary_work_id:
        abort(403)

    serializer = VoluntaryWorkSerializer([voluntary_work], many=True)
    json2 = {'voluntary_works': serializer.data}

    try:
        patch = jsonpatch.JsonPatch(data)
    except Exception as es:
        print es

    data = data['voluntary_works'][0]
    data.pop('links')
    data.pop('token')

    try:
        for key, value in data.iteritems():
            setattr(voluntary_work, key, value)
        db.session.commit()
    except Exception as es:
        abort(403)

    #return Response(mimetype='application/json', status=204)
    ser = _serialize([voluntary_work])

    response = jsonify(**ser)
    response.status_code = 200
    response.location = url_for('.get', id=voluntary_work.id)
    return response

def _get_links():
    return {
        'voluntary_works.municipality': {
            'href': (
                url_for('municipality.index', _external=True) +
                '/{voluntary_works.municipality}'
            ),
            'type': 'municipalities'
        },
        'voluntary_works.type': {
            'href': (
                url_for('type.index', _external=True) +
                '/{voluntary_works.type}'
            ),
            'type': 'types'
        }
    }

def _get_linked(voluntary_works):
    relationships = [
        ('type', 'types', 'id', VoluntaryWorkTypeSerializer),
        ('municipality', 'municipalities', 'code', MunicipalitySerializer),
    ]
    linked = {}
    for name, plural_name, primary_key, serializer_cls in relationships:
        items = set(getattr(work, name) for work in voluntary_works)
        items = sorted(items, key=operator.attrgetter(primary_key))
        serializer = serializer_cls(items, many=True)
        linked[plural_name] = serializer.data
    return linked

def _serialize(voluntary_works):
    serializer = VoluntaryWorkSerializer(voluntary_works, many=True)
    return {
        'links': _get_links(),
        'voluntary_works': serializer.data,
        'linked': _get_linked(voluntary_works)
    }

def _serialize_pagination(pagination):
    data = _serialize(pagination.items)
    data['meta'] = {
        'pagination': {
            'page': pagination.page,
            'pages': pagination.pages,
            'per_page': pagination.per_page,
            'total': pagination.total,
        }
    }
    return data
<|endoftext|>


<commit_before>import binascii
import hashlib
import json
import os
import shutil
import logging
import textwrap

from operator import itemgetter

from .cache import CacheError

try:
    from colorama import Fore
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()

from .project import evaluate_conditionals
from .project import ProjectDefinition

from .sources.download import Download
from .sources.directory import Directory
from .sources.git import GitRepository

from .cd import cd
from .override_environment import OverrideEnvironment
from .target import Target
from .filesystem import clean_directory
from .build_cache import BuildCache

from .process import command
from .projects import project_types

class Library:
    def __init__(self, needy, name, target=None, configuration=None, development_mode=False, build_caches=[]):
        self.needy = needy
        self.__name = name
        self.__target = target
        self.__configuration = configuration
        self.__directory = os.path.join(needy.needs_directory(), name)
        self.__development_mode = development_mode
        self.__build_caches = build_caches

    def configuration(self):
        return self.__configuration

    def name(self):
        return self.__name

    def target(self):
        return self.__target

    def project_configuration(self):
        return evaluate_conditionals(self.__configuration['project'] if 'project' in self.__configuration else dict(), self.target())

    def dependencies(self):
        str_or_list = self.configuration().get('dependencies', [])
        return str_or_list if isinstance(str_or_list, list) else [str_or_list]

    def string_format_variables(self):
        return {
            'build_directory': self.build_directory(),
            'platform': self.target().platform.identifier(),
            'architecture': self.target().architecture,
            'needs_file_directory': self.needy.path(),
            'build_concurrency': self.needy.build_concurrency(),
        }

    @staticmethod
    def additional_project_configuration_keys():
        """ the configuration keys that we handle here instead of in Project classes (usually because we need them before determining the project type) """
        return {'post-clean', 'configure-steps', 'environment', 'type', 'root'}

    def evaluate(self, str_or_list, **kwargs):
        l = [] if not str_or_list else (str_or_list if isinstance(str_or_list, list) else [str_or_list])
        variables = self.string_format_variables()
        variables.update(kwargs)
        return [str.format(**variables) for str in l]

    def clean_source(self):
        source = self.source()
        source.clean()

    def clean_build(self):
        clean_directory(self.build_directory())

    def initialize_source(self):
        self.clean_source()
        with OverrideEnvironment(self.__environment_overrides()):
            self.__post_clean()

    def synchronize_source(self):
        source = self.source()
        source.synchronize()

    def source(self):
        cfg = self.__configuration
        if 'download' in cfg:
            return Download(cfg['download'], cfg['checksum'], self.source_directory(), os.path.join(self.directory(), 'download'))
        if 'repository' in cfg:
            return GitRepository(cfg['repository'], cfg['commit'], self.source_directory())
        if 'directory' in cfg:
            return Directory(cfg['directory'] if os.path.isabs(cfg['directory']) else os.path.join(self.needy.path(), cfg['directory']), self.source_directory())
        raise ValueError('no source specified in configuration')

    def build(self):
        print('Building for %s %s' % (self.target().platform.identifier(), self.target().architecture))

        if ' ' in self.__directory:
            print(Fore.YELLOW + '[WARNING]' + Fore.RESET + ' The build path contains spaces. Some build systems don\'t '
                  'handle spaces well, so if you have problems, consider moving the project or using a symlink.')

        if not self.needy.parameters().force_build and not self.is_in_development_mode():
            if self.__load_cached_artifacts():
                return True

        if not self.is_in_development_mode():
            self.clean_source()

        with OverrideEnvironment(self.__environment_overrides()):
            if not self.is_in_development_mode():
                self.__post_clean()

            configuration = self.project_configuration()

            project = self.project(ProjectDefinition(self.target(), self.project_root(), configuration))
            if not project:
                raise RuntimeError('unknown project type')

            unrecognized_configuration_keys = set(configuration.keys()) - project.configuration_keys() - self.additional_project_configuration_keys()
            if len(unrecognized_configuration_keys):
                raise RuntimeError('unrecognized project configuration keys: {}'.format(', '.join(unrecognized_configuration_keys)))

            project.set_string_format_variables(**self.string_format_variables())

            if not self.is_in_development_mode():
                self.clean_build()

            self.__actualize(project)
            self.__write_build_status()
            self.__cache_artifacts()

        return True

    def __post_clean(self):
        configuration = self.project_configuration()
        post_clean_commands = configuration['post-clean'] if 'post-clean' in configuration else []
        with cd(self.project_root()):
            for cmd in self.evaluate(post_clean_commands):
                command(cmd)

    def __actualize(self, project):
        build_directory = self.build_directory()
        configuration = self.project_configuration()
        with cd(self.project_root()):
            try:
                project.setup()
                if 'configure-steps' in configuration:
                    project.run_commands(configuration['configure-steps'])
                else:
                    project.configure(build_directory)
                project.pre_build(build_directory)
                project.build(build_directory)
                project.post_build(build_directory)
                Library.make_pkgconfigs_relocatable(build_directory)
                if not os.path.exists(os.path.join(build_directory, 'lib', 'pkgconfig')):
                    self.generate_pkgconfig(build_directory, self.name())
            except:
                shutil.rmtree(build_directory)
                raise

    def __write_build_status(self):
        with open(self.build_status_path(), 'w') as status_file:
            if self.is_in_development_mode():
                status = {}
            else:
                status = {
                    'configuration': binascii.hexlify(self.configuration_hash()).decode()
                }
            json.dump(status, status_file)

    def __cache_artifacts(self):
        if self.__build_caches:
            try:
                self.__build_caches[0].store_artifacts(self.build_directory(), self.__cache_key())
            except:
                pass

    def __load_cached_artifacts(self):
        '''Returns True if there is a cache and it was able to load artifacts'''
        for c in self.__build_caches:
            try:
                c.load_artifacts(self.__cache_key(), self.build_directory())
                return True
            except (CacheError, IOError):
                pass
        return False

    def __cache_key(self):
        configuration_hash = binascii.hexlify(self.configuration_hash()).decode()
        path = os.path.relpath(self.build_directory(), self.needy.needs_directory())
        return os.path.join(path, configuration_hash)

    def __environment_overrides(self):
        configuration = self.project_configuration()
        overrides = self.target().platform.environment_overrides(self.target().architecture)
        if 'environment' in configuration:
            for key, value in configuration['environment'].items():
                overrides[key] = self.evaluate(value, current=overrides.get(key, os.environ.get(key, '')))[0]
        self.__log_environment_overrides(overrides)

        dependencies = self.dependencies()
        dependencies_to_provide = [dependency for dependency in dependencies if not self.needy.library_is_overridden(dependency)]
        if dependencies_to_provide:
            pkg_config_path = overrides['PKG_CONFIG_PATH'] if 'PKG_CONFIG_PATH' in overrides else os.environ.get('PKG_CONFIG_PATH', '')
            dependency_config_path = self.needy.pkg_config_path(self.target(), dependencies_to_provide)
            overrides['PKG_CONFIG_PATH'] = (dependency_config_path + ':' + pkg_config_path) if pkg_config_path and dependency_config_path else dependency_config_path

        return overrides

    def __log_environment_overrides(self, overrides, verbosity=logging.DEBUG):
        if overrides is not None and len(overrides) > 0:
            logging.log(verbosity, 'Overriding environment with new variables:')
            for k, v in overrides.items():
                logging.log(verbosity, '{}={}'.format(k, v))

    def is_in_development_mode(self):
        return self.__development_mode

    def is_up_to_date(self):
        if not os.path.isfile(self.build_status_path()) or self.is_in_development_mode():
            return False
        with open(self.build_status_path(), 'r') as status_file:
            status_text = status_file.read()
            if not status_text.strip():
                return False
            status = json.loads(status_text)
            if 'configuration' not in status or binascii.unhexlify(status['configuration']) != self.configuration_hash():
                return False
        return True

    def status_text(self):
        if self.is_in_development_mode():
            return 'dev mode'
        if self.is_up_to_date():
            return 'up-to-date'
        return 'out-of-date'

    def substatus_texts(self):
        ret = {}
        if self.is_in_development_mode():
            status = self.source().status_text()
            if status:
                ret[os.path.relpath(self.source_directory())] = '{}: {}'.format(self.source().identifier(), status)
            else:
                ret[os.path.relpath(self.source_directory())] = '{}'.format(self.source().identifier())
        return ret

    def directory(self):
        return self.__directory

    def build_directory(self):
        directory = os.path.join(self.__directory, 'build', self.target().platform.identifier(), self.target().architecture)
        suffix = self.configuration().get('build-directory-suffix')
        if suffix:
            directory = os.path.join(directory, suffix.lstrip(os.path.sep))
        return directory

    def build_status_path(self):
        return os.path.join(self.build_directory(), 'needy.status')

    def source_directory(self):
        return os.path.join(self.__directory, 'source')

    def project_root(self):
        configuration = self.project_configuration()
        return os.path.join(self.source_directory(), configuration['root']) if 'root' in configuration else self.source_directory()

    def include_path(self):
        return os.path.join(self.build_directory(), 'include')

    def library_path(self):
        return os.path.join(self.build_directory(), 'lib')

    def project(self, definition):
        candidates = project_types

        if 'type' in definition.configuration:
            for candidate in candidates:
                if candidate.identifier() == definition.configuration['type']:
                    missing_prerequisites = candidate.missing_prerequisites(definition, self.needy)
                    if len(missing_prerequisites) > 0:
                        raise RuntimeError('missing prerequisites for explicit project type: {}'.format(', '.join(missing_prerequisites)))
                    return candidate(definition, self.needy)
            raise RuntimeError('unknown project type')

        scores = [(len(set(definition.configuration.keys()) & set(c.configuration_keys())), c) for c in candidates]
        candidates = [candidate for score, candidate in sorted(scores, key=itemgetter(0), reverse=True)]

        logging.debug('project candidates ordered by number of valid configuration keys: {}'.format(', '.join([c.identifier() for c in candidates])))
        logging.debug('evaluating candidates in {}'.format(definition.directory))
        with cd(definition.directory):
            for candidate in candidates:
                valid, reasons = candidate.is_valid_project(definition, self.needy)
                missing_prerequisites = candidate.missing_prerequisites(definition, self.needy)
                logging.debug('project type determined {} be {}'.format('to' if valid else 'not to', candidate.identifier()))
                if isinstance(reasons, list):
                    for r in reasons:
                        logging.debug('  - {}'.format(r))
                else:
                    logging.debug('  - {}'.format(reasons))
                if valid:
                    if len(missing_prerequisites) > 0:
                        print(Fore.YELLOW + '[WARNING]' + Fore.RESET + ' Detected {} project, but the following prerequisites are missing: {}'.format(
                            candidate.identifier(), ', '.join(missing_prerequisites)
                        ))
                        continue
                    return candidate(definition, self.needy)

        raise RuntimeError('unknown project type')

    @classmethod
    def build_compatibility(cls):
        return 4

    def configuration_hash(self):
        hash = hashlib.sha256()

        platform_configuration_hash = self.target().platform.configuration_hash(self.target().architecture)
        if platform_configuration_hash:
            hash.update(platform_configuration_hash)

        configuration = self.__configuration.copy()
        configuration['project'] = self.project_configuration()

        hash.update(json.dumps({
            'build-compatibility': self.build_compatibility(),
            'configuration': configuration,
            'dependencies': [self.needy.library_configuration(self.target(), d) for d in (configuration['dependencies'] if 'dependencies' in configuration else [])],
        }, sort_keys=True).encode())

        return hash.digest()

    @staticmethod
    def generate_pkgconfig(prefix, library_name):
        libs = []

        lib_path = os.path.join(prefix, 'lib')

        if os.path.exists(lib_path):
            for file in os.listdir(lib_path):
                name, ext = os.path.splitext(file)
                if os.path.isfile(os.path.join(lib_path, file)) and name.startswith('lib') and len(name) > 3:
                    lib_name = name[3:]
                    if lib_name not in libs:
                        libs.append(lib_name)

        packages = {lib_name: [lib_name] for lib_name in libs}
        if library_name not in packages:
            packages[library_name] = libs

        pkgconfig_path = os.path.join(prefix, 'lib', 'pkgconfig')

        for name, libs in packages.items():
            pc_path = os.path.join(pkgconfig_path, name+'.pc')
            if not os.path.exists(pc_path):
                if not os.path.exists(pkgconfig_path):
                    os.makedirs(pkgconfig_path)
                with open(pc_path, 'w') as f:
                    f.write(textwrap.dedent("""\
                        # This file was automatically generated by Needy.
                        prefix=${{pcfiledir}}/../..
                        exec_prefix=${{prefix}}
                        libdir=${{exec_prefix}}/lib
                        includedir=${{prefix}}/include

                        Name: {name}
                        Version: {version}
                        Description: {name}
                        Libs: -L${{libdir}} {lflags}
                        Cflags: -I${{includedir}}
                    """.format(name=name, version=0, lflags=' '.join([('-l'+lib_name) for lib_name in libs]))))

    @staticmethod
    def make_pkgconfigs_relocatable(build_dir):
        if os.path.exists(build_dir):
            for dirname, subdirs, files in os.walk(os.path.join(build_dir, 'lib')):
                for filename in files:
                    if filename.endswith('.pc'):
                        pc = ''
                        with open(os.path.join(dirname, filename), 'r') as f:
                            pc = f.read().replace(build_dir, '${pcfiledir}/../..')
                        with open(os.path.join(dirname, filename), 'w+') as f:
                            f.write(pc)
<commit_msg>Don't cache dev-mode enabled needs<commit_after>import binascii
import hashlib
import json
import os
import shutil
import logging
import textwrap

from operator import itemgetter

from .cache import CacheError

try:
    from colorama import Fore
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()

from .project import evaluate_conditionals
from .project import ProjectDefinition

from .sources.download import Download
from .sources.directory import Directory
from .sources.git import GitRepository

from .cd import cd
from .override_environment import OverrideEnvironment
from .target import Target
from .filesystem import clean_directory
from .build_cache import BuildCache

from .process import command
from .projects import project_types

class Library:
    def __init__(self, needy, name, target=None, configuration=None, development_mode=False, build_caches=[]):
        self.needy = needy
        self.__name = name
        self.__target = target
        self.__configuration = configuration
        self.__directory = os.path.join(needy.needs_directory(), name)
        self.__development_mode = development_mode
        self.__build_caches = build_caches

    def configuration(self):
        return self.__configuration

    def name(self):
        return self.__name

    def target(self):
        return self.__target

    def project_configuration(self):
        return evaluate_conditionals(self.__configuration['project'] if 'project' in self.__configuration else dict(), self.target())

    def dependencies(self):
        str_or_list = self.configuration().get('dependencies', [])
        return str_or_list if isinstance(str_or_list, list) else [str_or_list]

    def string_format_variables(self):
        return {
            'build_directory': self.build_directory(),
            'platform': self.target().platform.identifier(),
            'architecture': self.target().architecture,
            'needs_file_directory': self.needy.path(),
            'build_concurrency': self.needy.build_concurrency(),
        }

    @staticmethod
    def additional_project_configuration_keys():
        """ the configuration keys that we handle here instead of in Project classes (usually because we need them before determining the project type) """
        return {'post-clean', 'configure-steps', 'environment', 'type', 'root'}

    def evaluate(self, str_or_list, **kwargs):
        l = [] if not str_or_list else (str_or_list if isinstance(str_or_list, list) else [str_or_list])
        variables = self.string_format_variables()
        variables.update(kwargs)
        return [str.format(**variables) for str in l]

    def clean_source(self):
        source = self.source()
        source.clean()

    def clean_build(self):
        clean_directory(self.build_directory())

    def initialize_source(self):
        self.clean_source()
        with OverrideEnvironment(self.__environment_overrides()):
            self.__post_clean()

    def synchronize_source(self):
        source = self.source()
        source.synchronize()

    def source(self):
        cfg = self.__configuration
        if 'download' in cfg:
            return Download(cfg['download'], cfg['checksum'], self.source_directory(), os.path.join(self.directory(), 'download'))
        if 'repository' in cfg:
            return GitRepository(cfg['repository'], cfg['commit'], self.source_directory())
        if 'directory' in cfg:
            return Directory(cfg['directory'] if os.path.isabs(cfg['directory']) else os.path.join(self.needy.path(), cfg['directory']), self.source_directory())
        raise ValueError('no source specified in configuration')

    def build(self):
        print('Building for %s %s' % (self.target().platform.identifier(), self.target().architecture))

        if ' ' in self.__directory:
            print(Fore.YELLOW + '[WARNING]' + Fore.RESET + ' The build path contains spaces. Some build systems don\'t '
                  'handle spaces well, so if you have problems, consider moving the project or using a symlink.')

        if not self.needy.parameters().force_build and not self.is_in_development_mode():
            if self.__load_cached_artifacts():
                return True

        if not self.is_in_development_mode():
            self.clean_source()

        with OverrideEnvironment(self.__environment_overrides()):
            if not self.is_in_development_mode():
                self.__post_clean()

            configuration = self.project_configuration()

            project = self.project(ProjectDefinition(self.target(), self.project_root(), configuration))
            if not project:
                raise RuntimeError('unknown project type')

            unrecognized_configuration_keys = set(configuration.keys()) - project.configuration_keys() - self.additional_project_configuration_keys()
            if len(unrecognized_configuration_keys):
                raise RuntimeError('unrecognized project configuration keys: {}'.format(', '.join(unrecognized_configuration_keys)))

            project.set_string_format_variables(**self.string_format_variables())

            if not self.is_in_development_mode():
                self.clean_build()

            self.__actualize(project)
            self.__write_build_status()
            if not self.is_in_development_mode():
                self.__cache_artifacts()

        return True

    def __post_clean(self):
        configuration = self.project_configuration()
        post_clean_commands = configuration['post-clean'] if 'post-clean' in configuration else []
        with cd(self.project_root()):
            for cmd in self.evaluate(post_clean_commands):
                command(cmd)

    def __actualize(self, project):
        build_directory = self.build_directory()
        configuration = self.project_configuration()
        with cd(self.project_root()):
            try:
                project.setup()
                if 'configure-steps' in configuration:
                    project.run_commands(configuration['configure-steps'])
                else:
                    project.configure(build_directory)
                project.pre_build(build_directory)
                project.build(build_directory)
                project.post_build(build_directory)
                Library.make_pkgconfigs_relocatable(build_directory)
                if not os.path.exists(os.path.join(build_directory, 'lib', 'pkgconfig')):
                    self.generate_pkgconfig(build_directory, self.name())
            except:
                shutil.rmtree(build_directory)
                raise

    def __write_build_status(self):
        with open(self.build_status_path(), 'w') as status_file:
            if self.is_in_development_mode():
                status = {}
            else:
                status = {
                    'configuration': binascii.hexlify(self.configuration_hash()).decode()
                }
            json.dump(status, status_file)

    def __cache_artifacts(self):
        if self.__build_caches:
            try:
                self.__build_caches[0].store_artifacts(self.build_directory(), self.__cache_key())
            except:
                pass

    def __load_cached_artifacts(self):
        '''Returns True if there is a cache and it was able to load artifacts'''
        for c in self.__build_caches:
            try:
                c.load_artifacts(self.__cache_key(), self.build_directory())
                return True
            except (CacheError, IOError):
                pass
        return False

    def __cache_key(self):
        configuration_hash = binascii.hexlify(self.configuration_hash()).decode()
        path = os.path.relpath(self.build_directory(), self.needy.needs_directory())
        return os.path.join(path, configuration_hash)

    def __environment_overrides(self):
        configuration = self.project_configuration()
        overrides = self.target().platform.environment_overrides(self.target().architecture)
        if 'environment' in configuration:
            for key, value in configuration['environment'].items():
                overrides[key] = self.evaluate(value, current=overrides.get(key, os.environ.get(key, '')))[0]
        self.__log_environment_overrides(overrides)

        dependencies = self.dependencies()
        dependencies_to_provide = [dependency for dependency in dependencies if not self.needy.library_is_overridden(dependency)]
        if dependencies_to_provide:
            pkg_config_path = overrides['PKG_CONFIG_PATH'] if 'PKG_CONFIG_PATH' in overrides else os.environ.get('PKG_CONFIG_PATH', '')
            dependency_config_path = self.needy.pkg_config_path(self.target(), dependencies_to_provide)
            overrides['PKG_CONFIG_PATH'] = (dependency_config_path + ':' + pkg_config_path) if pkg_config_path and dependency_config_path else dependency_config_path

        return overrides

    def __log_environment_overrides(self, overrides, verbosity=logging.DEBUG):
        if overrides is not None and len(overrides) > 0:
            logging.log(verbosity, 'Overriding environment with new variables:')
            for k, v in overrides.items():
                logging.log(verbosity, '{}={}'.format(k, v))

    def is_in_development_mode(self):
        return self.__development_mode

    def is_up_to_date(self):
        if not os.path.isfile(self.build_status_path()) or self.is_in_development_mode():
            return False
        with open(self.build_status_path(), 'r') as status_file:
            status_text = status_file.read()
            if not status_text.strip():
                return False
            status = json.loads(status_text)
            if 'configuration' not in status or binascii.unhexlify(status['configuration']) != self.configuration_hash():
                return False
        return True

    def status_text(self):
        if self.is_in_development_mode():
            return 'dev mode'
        if self.is_up_to_date():
            return 'up-to-date'
        return 'out-of-date'

    def substatus_texts(self):
        ret = {}
        if self.is_in_development_mode():
            status = self.source().status_text()
            if status:
                ret[os.path.relpath(self.source_directory())] = '{}: {}'.format(self.source().identifier(), status)
            else:
                ret[os.path.relpath(self.source_directory())] = '{}'.format(self.source().identifier())
        return ret

    def directory(self):
        return self.__directory

    def build_directory(self):
        directory = os.path.join(self.__directory, 'build', self.target().platform.identifier(), self.target().architecture)
        suffix = self.configuration().get('build-directory-suffix')
        if suffix:
            directory = os.path.join(directory, suffix.lstrip(os.path.sep))
        return directory

    def build_status_path(self):
        return os.path.join(self.build_directory(), 'needy.status')

    def source_directory(self):
        return os.path.join(self.__directory, 'source')

    def project_root(self):
        configuration = self.project_configuration()
        return os.path.join(self.source_directory(), configuration['root']) if 'root' in configuration else self.source_directory()

    def include_path(self):
        return os.path.join(self.build_directory(), 'include')

    def library_path(self):
        return os.path.join(self.build_directory(), 'lib')

    def project(self, definition):
        candidates = project_types

        if 'type' in definition.configuration:
            for candidate in candidates:
                if candidate.identifier() == definition.configuration['type']:
                    missing_prerequisites = candidate.missing_prerequisites(definition, self.needy)
                    if len(missing_prerequisites) > 0:
                        raise RuntimeError('missing prerequisites for explicit project type: {}'.format(', '.join(missing_prerequisites)))
                    return candidate(definition, self.needy)
            raise RuntimeError('unknown project type')

        scores = [(len(set(definition.configuration.keys()) & set(c.configuration_keys())), c) for c in candidates]
        candidates = [candidate for score, candidate in sorted(scores, key=itemgetter(0), reverse=True)]

        logging.debug('project candidates ordered by number of valid configuration keys: {}'.format(', '.join([c.identifier() for c in candidates])))
        logging.debug('evaluating candidates in {}'.format(definition.directory))
        with cd(definition.directory):
            for candidate in candidates:
                valid, reasons = candidate.is_valid_project(definition, self.needy)
                missing_prerequisites = candidate.missing_prerequisites(definition, self.needy)
                logging.debug('project type determined {} be {}'.format('to' if valid else 'not to', candidate.identifier()))
                if isinstance(reasons, list):
                    for r in reasons:
                        logging.debug('  - {}'.format(r))
                else:
                    logging.debug('  - {}'.format(reasons))
                if valid:
                    if len(missing_prerequisites) > 0:
                        print(Fore.YELLOW + '[WARNING]' + Fore.RESET + ' Detected {} project, but the following prerequisites are missing: {}'.format(
                            candidate.identifier(), ', '.join(missing_prerequisites)
                        ))
                        continue
                    return candidate(definition, self.needy)

        raise RuntimeError('unknown project type')

    @classmethod
    def build_compatibility(cls):
        return 4

    def configuration_hash(self):
        hash = hashlib.sha256()

        platform_configuration_hash = self.target().platform.configuration_hash(self.target().architecture)
        if platform_configuration_hash:
            hash.update(platform_configuration_hash)

        configuration = self.__configuration.copy()
        configuration['project'] = self.project_configuration()

        hash.update(json.dumps({
            'build-compatibility': self.build_compatibility(),
            'configuration': configuration,
            'dependencies': [self.needy.library_configuration(self.target(), d) for d in (configuration['dependencies'] if 'dependencies' in configuration else [])],
        }, sort_keys=True).encode())

        return hash.digest()

    @staticmethod
    def generate_pkgconfig(prefix, library_name):
        libs = []

        lib_path = os.path.join(prefix, 'lib')

        if os.path.exists(lib_path):
            for file in os.listdir(lib_path):
                name, ext = os.path.splitext(file)
                if os.path.isfile(os.path.join(lib_path, file)) and name.startswith('lib') and len(name) > 3:
                    lib_name = name[3:]
                    if lib_name not in libs:
                        libs.append(lib_name)

        packages = {lib_name: [lib_name] for lib_name in libs}
        if library_name not in packages:
            packages[library_name] = libs

        pkgconfig_path = os.path.join(prefix, 'lib', 'pkgconfig')

        for name, libs in packages.items():
            pc_path = os.path.join(pkgconfig_path, name+'.pc')
            if not os.path.exists(pc_path):
                if not os.path.exists(pkgconfig_path):
                    os.makedirs(pkgconfig_path)
                with open(pc_path, 'w') as f:
                    f.write(textwrap.dedent("""\
                        # This file was automatically generated by Needy.
                        prefix=${{pcfiledir}}/../..
                        exec_prefix=${{prefix}}
                        libdir=${{exec_prefix}}/lib
                        includedir=${{prefix}}/include

                        Name: {name}
                        Version: {version}
                        Description: {name}
                        Libs: -L${{libdir}} {lflags}
                        Cflags: -I${{includedir}}
                    """.format(name=name, version=0, lflags=' '.join([('-l'+lib_name) for lib_name in libs]))))

    @staticmethod
    def make_pkgconfigs_relocatable(build_dir):
        if os.path.exists(build_dir):
            for dirname, subdirs, files in os.walk(os.path.join(build_dir, 'lib')):
                for filename in files:
                    if filename.endswith('.pc'):
                        pc = ''
                        with open(os.path.join(dirname, filename), 'r') as f:
                            pc = f.read().replace(build_dir, '${pcfiledir}/../..')
                        with open(os.path.join(dirname, filename), 'w+') as f:
                            f.write(pc)
<|endoftext|>


<commit_before><commit_msg>add test for nomination message<commit_after>import pytest
from plenum.common.types import Nomination
from collections import OrderedDict
from plenum.common.messages.fields import NonNegativeNumberField, \
    LedgerIdField, MerkleRootField, IterableField, NonEmptyStringField

EXPECTED_ORDERED_FIELDS = OrderedDict([
    ("name", NonEmptyStringField),
    ("instId", NonNegativeNumberField),
    ("viewNo", NonNegativeNumberField),
    ("ordSeqNo", NonNegativeNumberField),
])

def test_hash_expected_type():
    assert Nomination.typename == "NOMINATE"

def test_has_expected_fields():
    actual_field_names = OrderedDict(Nomination.schema).keys()
    assert actual_field_names == EXPECTED_ORDERED_FIELDS.keys()

def test_has_expected_validators():
    schema = dict(Nomination.schema)
    for field, validator in EXPECTED_ORDERED_FIELDS.items():
        assert isinstance(schema[field], validator)
<|endoftext|>