<commit_before>from __future__ import division
import numpy as np
from numdifftools.extrapolation import EPS
from collections import namedtuple

_STATE = namedtuple('State', ['x', 'method', 'n', 'order'])

def make_exact(h):
    """Make sure h is an exact representable number

    This is important when calculating numerical derivatives and is
    accomplished by adding 1.0 and then subtracting 1.0.
    """
    return (h + 1.0) - 1.0

def valarray(shape, value=np.NaN, typecode=None):
    """Return an array of all value."""
    if typecode is None:
        typecode = bool
    out = np.ones(shape, dtype=typecode) * value

    if not isinstance(out, np.ndarray):
        out = np.asarray(out)
    return out

def nominal_step(x=None):
    """Return nominal step"""
    if x is None:
        return 1.0
    return np.log1p(np.abs(x)).clip(min=1.0)

def base_step(scale):
    """Return base_step = EPS ** (1. / scale)"""
    return EPS ** (1. / scale)

# def default_base_step(x, scale, epsilon=None):
#     if epsilon is None:
#         h = base_step(scale) * nominal_step(x)
#     else:
#         h = valarray(x.shape, value=epsilon)
#     return h

def default_scale(method='forward', n=1, order=2):
    high_order = int(n > 1 or order >= 4)
    order2 = max(order // 2 - 1, 0)
    n4 = n // 4

    return (dict(multicomplex=1.35, complex=1.35).get(method, 2.5) +
            int((n - 1)) * dict(multicomplex=0, complex=0.0).get(method, 1.3) +
            order2 * dict(central=3, forward=2, backward=2).get(method, 0) +
            (n % 4 == 1) * high_order * dict(complex=3.65 + n4 * (5 + 1.5**n4)
                                             ).get(method, 0) +
            (n % 4 == 3) * dict(complex=3.65 * 2 + n4 * (5 + 2.1**n4)
                                ).get(method, 0) +
            (n % 4 == 2) * dict(complex=3.65 + n4 * (5 + 1.7**n4)
                                ).get(method, 0) +
            (n % 4 == 0) * dict(complex=n4 * (10 + 1.5 * int(n > 10))
                                ).get(method, 0))

class BasicMaxStepGenerator(object):

    """
    Generates a sequence of steps of decreasing magnitude

    where
        steps = base_step * step_ratio ** (-i + offset), i=0, 1,.., num_steps-1.

    Parameters
    ----------
    base_step : float, array-like.
       Defines the start step, i.e., maximum step
    step_ratio : real scalar.
        Ratio between sequential steps generated.  Note: Ratio > 1
    num_steps : scalar integer.
        defines number of steps generated.
    offset : real scalar, optional, default 0
        offset to the base step

    Example
    -------
    >>> step_gen = BasicMaxStepGenerator(base_step=2.0, step_ratio=2, num_steps=4)
    >>> [s for s in step_gen()]
    [2.0, 1.0, 0.5, 0.25]

    """

    _sign = -1

    def __init__(self, base_step, step_ratio, num_steps, offset=0):
        self.base_step = base_step
        self.step_ratio = step_ratio
        self.num_steps = num_steps
        self.offset = offset

    def _range(self):
        return range(self.num_steps)

    def __call__(self):
        base_step, step_ratio = self.base_step, self.step_ratio
        sgn, offset = self._sign, self.offset
        for i in self._range():
            step = base_step * step_ratio ** (sgn * i + offset)
            if (np.abs(step) > 0).all():
                yield step

class BasicMinStepGenerator(BasicMaxStepGenerator):

    """
    Generates a sequence of steps of decreasing magnitude

    where
        steps = base_step * step_ratio ** (i + offset), i=num_steps-1,... 1, 0.

    Parameters
    ----------
    base_step : float, array-like.
       Defines the end step, i.e., minimum step
    step_ratio : real scalar.
        Ratio between sequential steps generated.  Note: Ratio > 1
    num_steps : scalar integer.
        defines number of steps generated.
    offset : real scalar, optional, default 0
        offset to the base step

    Example
    -------
    >>> step_gen = BasicMinStepGenerator(base_step=0.25, step_ratio=2, num_steps=4)
    >>> [s for s in step_gen()]
    [2.0, 1.0, 0.5, 0.25]

    """

    _sign = 1

    def _range(self):
        return range(self.num_steps - 1, -1, -1)

class MinStepGenerator(object):

    """
    Generates a sequence of steps

    where
        steps = step_nom * base_step * step_ratio ** (i + offset)
    for  i = num_steps-1,... 1, 0.

    Parameters
    ----------
    base_step : float, array-like, optional
        Defines the minimum step, if None, the value is set to EPS**(1/scale)
    step_ratio : real scalar, optional, default 2
        Ratio between sequential steps generated.
        Note: Ratio > 1
        If None then step_ratio is 2 for n=1 otherwise step_ratio is 1.6
    num_steps : scalar integer, optional, default  min_num_steps + num_extrap
        defines number of steps generated. It should be larger than
        min_num_steps = (n + order - 1) / fact where fact is 1, 2 or 4 depending
        on differentiation method used.
    step_nom :  default maximum(log(1+|x|), 1)
        Nominal step where x is supplied at runtime through the __call__ method.
    offset : real scalar, optional, default 0
        offset to the base step
    num_extrap : scalar integer
        num_extrap
    check_num_steps : boolean
        If True make sure num_steps larger than the minimum required steps.
    use_exact_steps : boolean
        If true make sure exact steps are generated
    scale : real scalar, optional
        scale used in base step. If not None it will override the default
        computed with the default_scale function.
    """

    _step_generator = BasicMinStepGenerator

    def __init__(self, base_step=None, step_ratio=2.0, num_steps=None,
                 step_nom=None, offset=0, num_extrap=0, use_exact_steps=True,
                 check_num_steps=True, scale=None):
        self.base_step = base_step
        self.step_nom = step_nom
        self.num_steps = num_steps
        self.step_ratio = step_ratio
        self.offset = offset
        self.num_extrap = num_extrap
        self.check_num_steps = check_num_steps
        self.use_exact_steps = use_exact_steps
        self.scale = scale

        self._state = _STATE(x=np.asarray(1), method='forward', n=1, order=2)

    def __repr__(self):
        class_name = self.__class__.__name__
        kwds = ['{0!s}={1!s}'.format(name, str(getattr(self, name)))
                for name in self.__dict__.keys()]
        return """{0!s}({1!s})""".format(class_name, ','.join(kwds))

    @property
    def scale(self):
        if self._scale is None:
            _x, method, n, order = self._state
            return default_scale(method, n, order)
        return self._scale

    @scale.setter
    def scale(self, scale):
        self._scale = scale

    @property
    def base_step(self):
        if self._base_step is None:
            return base_step(self.scale)
        return self._base_step

    @base_step.setter
    def base_step(self, base_step):
        self._base_step = base_step

    @property
    def min_num_steps(self):
        _x, method, n, order = self._state
        num_steps = int(n + order - 1)
        if method in ['central', 'central2', 'complex', 'multicomplex']:
            step = 4 if method == 'complex' and (n > 1 or order >= 4) else 2
            num_steps = num_steps // step
        return max(num_steps, 1)

    @property
    def num_steps(self):
        min_num_steps = self.min_num_steps
        if self._num_steps is not None:
            num_steps = int(self._num_steps)
            if self.check_num_steps:
                num_steps = max(num_steps, min_num_steps)
            return num_steps
        return min_num_steps + int(self.num_extrap)

    @num_steps.setter
    def num_steps(self, num_steps):
        self._num_steps = num_steps

    @property
    def step_ratio(self,):
        step_ratio = self._step_ratio
        if step_ratio is None:
            step_ratio = {1: 2.0}.get(self._state.n, 1.6)
        return float(step_ratio)

    @step_ratio.setter
    def step_ratio(self, step_ratio):
        self._step_ratio = step_ratio

    @property
    def step_nom(self):
        x = self._state.x
        if self._step_nom is None:
            return nominal_step(x)
        return valarray(x.shape, value=self._step_nom)

    @step_nom.setter
    def step_nom(self, step_nom):
        self._step_nom = step_nom

    def step_generator_function(self, x, method='forward', n=1, order=2):
        self._state = _STATE(np.asarray(x), method, n, order)
        base_step, step_ratio = self.base_step * self.step_nom, self.step_ratio
        if self.use_exact_steps:
            base_step = make_exact(base_step)
            step_ratio = make_exact(step_ratio)
        return self._step_generator(base_step=base_step,
                                    step_ratio=step_ratio,
                                    num_steps=self.num_steps,
                                    offset=self.offset)

    def __call__(self, x, method='forward', n=1, order=2):
        step_generator = self.step_generator_function(x, method, n, order)
        return step_generator()

class MaxStepGenerator(MinStepGenerator):

    """
    Generates a sequence of steps

    where
        steps = step_nom * base_step * step_ratio ** (-i + offset)
    for  i = 0, 1, ..., num_steps-1.

    Parameters
    ----------
    base_step : float, array-like, default 2.0
        Defines the maximum step, if None, the value is set to EPS**(1/scale)
    step_ratio : real scalar, optional, default 2
        Ratio between sequential steps generated.
        Note: Ratio > 1
        If None then step_ratio is 2 for n=1 otherwise step_ratio is 1.6
    num_steps : scalar integer, optional, default  min_num_steps + num_extrap
        defines number of steps generated. It should be larger than
        min_num_steps = (n + order - 1) / fact where fact is 1, 2 or 4 depending
        on differentiation method used.
    step_nom :  default maximum(log(1+|x|), 1)
        Nominal step where x is supplied at runtime through the __call__ method.
    offset : real scalar, optional, default 0
        offset to the base step
    num_extrap : scalar integer
        num_extrap
    check_num_steps : boolean
        If True make sure num_steps larger than the minimum required steps.
    use_exact_steps : boolean
        If true make sure exact steps are generated
    scale : real scalar, default 500
        scale used in base step.
    """

    _step_generator = BasicMaxStepGenerator

    def __init__(self, base_step=2.0, step_ratio=2.0, num_steps=15,
                 step_nom=None, offset=0, num_extrap=0,
                 use_exact_steps=False, check_num_steps=True, scale=500):
        super(MaxStepGenerator,
              self).__init__(base_step=base_step, step_ratio=step_ratio,
                             num_steps=num_steps, step_nom=step_nom,
                             offset=offset, num_extrap=num_extrap,
                             use_exact_steps=use_exact_steps,
                             check_num_steps=check_num_steps, scale=scale)

if __name__ == '__main__':
    from numdifftools.testing import test_docstrings
    test_docstrings()
<commit_msg>Simplified default_scale<commit_after>from __future__ import division
import numpy as np
from numdifftools.extrapolation import EPS
from collections import namedtuple

_STATE = namedtuple('State', ['x', 'method', 'n', 'order'])

def make_exact(h):
    """Make sure h is an exact representable number

    This is important when calculating numerical derivatives and is
    accomplished by adding 1.0 and then subtracting 1.0.
    """
    return (h + 1.0) - 1.0

def valarray(shape, value=np.NaN, typecode=None):
    """Return an array of all value."""
    if typecode is None:
        typecode = bool
    out = np.ones(shape, dtype=typecode) * value

    if not isinstance(out, np.ndarray):
        out = np.asarray(out)
    return out

def nominal_step(x=None):
    """Return nominal step"""
    if x is None:
        return 1.0
    return np.log1p(np.abs(x)).clip(min=1.0)

def base_step(scale):
    """Return base_step = EPS ** (1. / scale)"""
    return EPS ** (1. / scale)

# def default_base_step(x, scale, epsilon=None):
#     if epsilon is None:
#         h = base_step(scale) * nominal_step(x)
#     else:
#         h = valarray(x.shape, value=epsilon)
#     return h

def default_scale(method='forward', n=1, order=2):
    high_order = int(n > 1 or order >= 4)
    order2 = max(order // 2 - 1, 0)
    n4 = n // 4
    n_mod_4 = n % 4
    c = ([n4 * (10 + 1.5 * int(n > 10)),
          3.65 + n4 * (5 + 1.5**n4),
          3.65 + n4 * (5 + 1.7**n4),
          7.30 + n4 * (5 + 2.1**n4)][n_mod_4]) if high_order else 0

    return (dict(multicomplex=1.35, complex=1.35+c).get(method, 2.5) +
            int((n - 1)) * dict(multicomplex=0, complex=0.0).get(method, 1.3) +
            order2 * dict(central=3, forward=2, backward=2).get(method, 0))

class BasicMaxStepGenerator(object):

    """
    Generates a sequence of steps of decreasing magnitude

    where
        steps = base_step * step_ratio ** (-i + offset), i=0, 1,.., num_steps-1.

    Parameters
    ----------
    base_step : float, array-like.
       Defines the start step, i.e., maximum step
    step_ratio : real scalar.
        Ratio between sequential steps generated.  Note: Ratio > 1
    num_steps : scalar integer.
        defines number of steps generated.
    offset : real scalar, optional, default 0
        offset to the base step

    Example
    -------
    >>> step_gen = BasicMaxStepGenerator(base_step=2.0, step_ratio=2, num_steps=4)
    >>> [s for s in step_gen()]
    [2.0, 1.0, 0.5, 0.25]

    """

    _sign = -1

    def __init__(self, base_step, step_ratio, num_steps, offset=0):
        self.base_step = base_step
        self.step_ratio = step_ratio
        self.num_steps = num_steps
        self.offset = offset

    def _range(self):
        return range(self.num_steps)

    def __call__(self):
        base_step, step_ratio = self.base_step, self.step_ratio
        sgn, offset = self._sign, self.offset
        for i in self._range():
            step = base_step * step_ratio ** (sgn * i + offset)
            if (np.abs(step) > 0).all():
                yield step

class BasicMinStepGenerator(BasicMaxStepGenerator):

    """
    Generates a sequence of steps of decreasing magnitude

    where
        steps = base_step * step_ratio ** (i + offset), i=num_steps-1,... 1, 0.

    Parameters
    ----------
    base_step : float, array-like.
       Defines the end step, i.e., minimum step
    step_ratio : real scalar.
        Ratio between sequential steps generated.  Note: Ratio > 1
    num_steps : scalar integer.
        defines number of steps generated.
    offset : real scalar, optional, default 0
        offset to the base step

    Example
    -------
    >>> step_gen = BasicMinStepGenerator(base_step=0.25, step_ratio=2, num_steps=4)
    >>> [s for s in step_gen()]
    [2.0, 1.0, 0.5, 0.25]

    """

    _sign = 1

    def _range(self):
        return range(self.num_steps - 1, -1, -1)

class MinStepGenerator(object):

    """
    Generates a sequence of steps

    where
        steps = step_nom * base_step * step_ratio ** (i + offset)
    for  i = num_steps-1,... 1, 0.

    Parameters
    ----------
    base_step : float, array-like, optional
        Defines the minimum step, if None, the value is set to EPS**(1/scale)
    step_ratio : real scalar, optional, default 2
        Ratio between sequential steps generated.
        Note: Ratio > 1
        If None then step_ratio is 2 for n=1 otherwise step_ratio is 1.6
    num_steps : scalar integer, optional, default  min_num_steps + num_extrap
        defines number of steps generated. It should be larger than
        min_num_steps = (n + order - 1) / fact where fact is 1, 2 or 4 depending
        on differentiation method used.
    step_nom :  default maximum(log(1+|x|), 1)
        Nominal step where x is supplied at runtime through the __call__ method.
    offset : real scalar, optional, default 0
        offset to the base step
    num_extrap : scalar integer
        num_extrap
    check_num_steps : boolean
        If True make sure num_steps larger than the minimum required steps.
    use_exact_steps : boolean
        If true make sure exact steps are generated
    scale : real scalar, optional
        scale used in base step. If not None it will override the default
        computed with the default_scale function.
    """

    _step_generator = BasicMinStepGenerator

    def __init__(self, base_step=None, step_ratio=2.0, num_steps=None,
                 step_nom=None, offset=0, num_extrap=0, use_exact_steps=True,
                 check_num_steps=True, scale=None):
        self.base_step = base_step
        self.step_nom = step_nom
        self.num_steps = num_steps
        self.step_ratio = step_ratio
        self.offset = offset
        self.num_extrap = num_extrap
        self.check_num_steps = check_num_steps
        self.use_exact_steps = use_exact_steps
        self.scale = scale

        self._state = _STATE(x=np.asarray(1), method='forward', n=1, order=2)

    def __repr__(self):
        class_name = self.__class__.__name__
        kwds = ['{0!s}={1!s}'.format(name, str(getattr(self, name)))
                for name in self.__dict__.keys()]
        return """{0!s}({1!s})""".format(class_name, ','.join(kwds))

    @property
    def scale(self):
        if self._scale is None:
            _x, method, n, order = self._state
            return default_scale(method, n, order)
        return self._scale

    @scale.setter
    def scale(self, scale):
        self._scale = scale

    @property
    def base_step(self):
        if self._base_step is None:
            return base_step(self.scale)
        return self._base_step

    @base_step.setter
    def base_step(self, base_step):
        self._base_step = base_step

    @property
    def min_num_steps(self):
        _x, method, n, order = self._state
        num_steps = int(n + order - 1)
        if method in ['central', 'central2', 'complex', 'multicomplex']:
            step = 4 if method == 'complex' and (n > 1 or order >= 4) else 2
            num_steps = num_steps // step
        return max(num_steps, 1)

    @property
    def num_steps(self):
        min_num_steps = self.min_num_steps
        if self._num_steps is not None:
            num_steps = int(self._num_steps)
            if self.check_num_steps:
                num_steps = max(num_steps, min_num_steps)
            return num_steps
        return min_num_steps + int(self.num_extrap)

    @num_steps.setter
    def num_steps(self, num_steps):
        self._num_steps = num_steps

    @property
    def step_ratio(self,):
        step_ratio = self._step_ratio
        if step_ratio is None:
            step_ratio = {1: 2.0}.get(self._state.n, 1.6)
        return float(step_ratio)

    @step_ratio.setter
    def step_ratio(self, step_ratio):
        self._step_ratio = step_ratio

    @property
    def step_nom(self):
        x = self._state.x
        if self._step_nom is None:
            return nominal_step(x)
        return valarray(x.shape, value=self._step_nom)

    @step_nom.setter
    def step_nom(self, step_nom):
        self._step_nom = step_nom

    def step_generator_function(self, x, method='forward', n=1, order=2):
        self._state = _STATE(np.asarray(x), method, n, order)
        base_step, step_ratio = self.base_step * self.step_nom, self.step_ratio
        if self.use_exact_steps:
            base_step = make_exact(base_step)
            step_ratio = make_exact(step_ratio)
        return self._step_generator(base_step=base_step,
                                    step_ratio=step_ratio,
                                    num_steps=self.num_steps,
                                    offset=self.offset)

    def __call__(self, x, method='forward', n=1, order=2):
        step_generator = self.step_generator_function(x, method, n, order)
        return step_generator()

class MaxStepGenerator(MinStepGenerator):

    """
    Generates a sequence of steps

    where
        steps = step_nom * base_step * step_ratio ** (-i + offset)
    for  i = 0, 1, ..., num_steps-1.

    Parameters
    ----------
    base_step : float, array-like, default 2.0
        Defines the maximum step, if None, the value is set to EPS**(1/scale)
    step_ratio : real scalar, optional, default 2
        Ratio between sequential steps generated.
        Note: Ratio > 1
        If None then step_ratio is 2 for n=1 otherwise step_ratio is 1.6
    num_steps : scalar integer, optional, default  min_num_steps + num_extrap
        defines number of steps generated. It should be larger than
        min_num_steps = (n + order - 1) / fact where fact is 1, 2 or 4 depending
        on differentiation method used.
    step_nom :  default maximum(log(1+|x|), 1)
        Nominal step where x is supplied at runtime through the __call__ method.
    offset : real scalar, optional, default 0
        offset to the base step
    num_extrap : scalar integer
        num_extrap
    check_num_steps : boolean
        If True make sure num_steps larger than the minimum required steps.
    use_exact_steps : boolean
        If true make sure exact steps are generated
    scale : real scalar, default 500
        scale used in base step.
    """

    _step_generator = BasicMaxStepGenerator

    def __init__(self, base_step=2.0, step_ratio=2.0, num_steps=15,
                 step_nom=None, offset=0, num_extrap=0,
                 use_exact_steps=False, check_num_steps=True, scale=500):
        super(MaxStepGenerator,
              self).__init__(base_step=base_step, step_ratio=step_ratio,
                             num_steps=num_steps, step_nom=step_nom,
                             offset=offset, num_extrap=num_extrap,
                             use_exact_steps=use_exact_steps,
                             check_num_steps=check_num_steps, scale=scale)

if __name__ == '__main__':
    from numdifftools.testing import test_docstrings
    test_docstrings()
<|endoftext|>


<commit_before>
import inspect

from .properties import (BaseProperty, HasProps, Instance, Enum, Float, Int,
        Color, Percent, Size, Bool, Pattern, Align, Angle, String)

from .objects import PlotObject, Viewable

# Size is a way to preserve a data-space-related metric all the way until
#   render time, when the screen dimensions are known
# Align may be "center", "min", "max", or "jitter(func)" where func is name
#   of a random distribution to draw random samples from. Defaults to uniform
#   but gaussian could certainly be useful.

#class Scene(object):
#    """ A set of related plots, each with their own panel(s) and each
#    panel representing some coherent coordinate space onto which Glyphs
#    are positioned and rendered.
#    """

class DataSpec(BaseProperty):
    """ Because the BokehJS glyphs support a fixed value or a named
    field for most data fields, we capture that in this property.

    We mirror the JS convention in this Python descriptor.  There are
    multiple ways to set a DataSpec, illustrated below with comments
    and example code.

    class Foo(HasProps):
        x = DataSpec("x", units="data")

    f = Foo()
    f.x = "fieldname"   # Sets x to a field named "fieldname"
    f.x = 12            # Sets x to a default value of 12
    f.x = {"name": "foo", "default": 12}

    Just reading out "f.x" will always retrieve what has been set.
    """
    # TODO: Check to see if the "units" field hasn't been removed...

    def __init__(self, field=None, units="data", default=None):
        """
        Parameters
        ==========
        **field** is the string name of a data column to look up.
        **units** is either "data" or "screen"
        **default** is the default value to use if a datapoint is
        missing the field specified in **name**

        If a constant value is desired, then leave **field** as None and
        set **default** to the fixed value.
        """
        # Don't use .name because the HasProps metaclass uses that to
        # store the attribute name on this descriptor.
        self.field = field
        self.units = units
        self.default = default

    @classmethod
    def autocreate(cls, name=None):
        # In this case, use the name the user assigned this DataSpec to
        # as the default field name.
        d = cls(field=name)
        return d

    def __get__(self, obj, cls=None):
        # It's kind of an open question what we should return here if the
        # user hasn't set anything yet.  We use our best heuristic and
        # return a name, a default, or a dict with both, depending on
        # what is None.
        if hasattr(obj, "_"+self.name):
            return getattr(obj, "_"+self.name)
        else:
            if self.field is None:
                return self.default
            elif self.default is None:
                return self.field
            else:
                return {"field": self.field, "default": self.default}

    def __set__(self, obj, value):
        setattr(obj, "_"+self.name, value)

    def __delete__(self, obj):
        if hasattr(obj, self.name + "_dict"):
            delattr(obj, self.name + "_dict")
        BaseProperty.__delete__(self, obj)

    def to_dict(self, obj):
        # Build the complete dict
        value = getattr(obj, "_"+self.name, self.default)
        if type(value) == str:
            d = {"name": value, "units": self.units, "default": self.default}
        elif isinstance(value, dict):
            d = {"name": self.field, "units": self.units, "default": self.default}
            d.update(value)
        else:
            # Assume value is a numeric type and is the default value.
            # We explicitly set the field name to None.
            d = {"name": None, "units": self.units, "default": value}
        return d

class FillProps(HasProps):
    """ Mirrors the BokehJS properties.fill_properties class """
    fill = Color("gray")
    fill_alpha = Percent(1.0)

class LineProps(HasProps):
    """ Mirrors the BokehJS properties.line_properties class """
    line_color = Color("red")
    line_width = Size(1)
    line_alpha = Percent(1.0)
    line_join = String("miter")
    line_cap = String("butt")
    line_dash = Pattern
    line_dash_offset = Int(0)

class TextProps(HasProps):
    """ Mirrors the BokehJS properties.text_properties class """
    text_font = String
    text_font_size = Int(10)
    text_font_style = Enum("normal", "italic", "bold")
    text_color = Color("black")
    text_alpha = Percent(1.0)
    text_align = Enum("left", "right", "center")
    text_baseline = Enum("top", "middle", "bottom")

class MetaGlyph(Viewable):
    """ Handles DataSpecs and other special attribute processing that Glyph
    subclasses need.

    I contemplated making this a class decorator, or monkeypatching the
    Viewable metaclass, but that all seemed more brittle than just
    making a sub-metaclass.
    """

    def __new__(cls, class_name, bases, class_dict):
        # This is a little unorthodox. Usually metaclasses munge the class_dict
        # and attributes before handing off to the base metaclass ctor. In our
        # case, we want the MetaHasProps.__new__ to handle all the machinery
        # of autocreating Property declarations and whatnot, and so we just
        # muck with the class after it's been created.  I've not seen it done
        # this way elsewhere, but it shouldn't be a problem.

        newcls = super(MetaGlyph,cls).__new__(cls, class_name, bases, class_dict)
        dataspecs = {}
        for attr,val in newcls.__dict__.iteritems():
            if isinstance(val, DataSpec):
                dataspecs[attr] = val
        setattr(newcls, "_dataspecs", dataspecs)
        return newcls

class Glyph(PlotObject):
    """ Base class for all glyphs/marks/geoms/whatever-you-call-'em in Bokeh.
    """

    __metaclass__ = MetaGlyph

    # The name that will be used for the 'type' field of the glyph when we
    # create the JS dict representation of this glyph.  "glyph" is not a valid
    # glyph type; it's just a placeholder.
    glyphtype = "glyph"

    # Common attributes for all glyphs
    #color = Color
    #alpha = Percent
    visible = Bool
    margin = Size   # the amount of desired space around this glyph
    halign = Align  # when there is horizontal wiggle room (e.g. categorical)
    valign = Align  # when there is vertical wiggle room

    radius_units = Enum("screen", "data")
    length_units = Enum("screen", "data")
    angle_units = Enum("deg", "rad")
    start_angle_units = Enum("deg", "rad")
    end_angle_units = Enum("deg", "rad")

    def to_glyphspec(self):
        """ Returns a dict mapping attributes to values, that is amenable for
        inclusion in a GlyphRenderer definition.
        """
        d = self.vm_props(withvalues=True)
        d["type"] = d.pop("glyphtype")
        # Iterate over all the DataSpec properties and convert them, using the
        # fact that DataSpecs store the dict-ified version on the object.
        for attrname, dspec in self.dataspecs().iteritems():
            d[attrname] = dspec.to_dict(self)
        return d

    def dataspecs(self):
        dataspecs = {}
        for cls in reversed(inspect.getmro(self.__class__)):
            if isinstance(cls, MetaGlyph):
                dataspecs.update(cls._dataspecs)
        return dataspecs

class Marker(Glyph, FillProps, LineProps):
    """ Base class for glyphs which are just simple markers placed at (x,y)
    locations.
    """

    x = DataSpec
    y = DataSpec
    #size = Size

    # TODO: Remove this and use the color attributes in Glyph instead
    #stroke_color = Color
    #fill_pattern = Pattern
    #shape = Enum("circle", "dot", "square", "tri", "diamond", "x", "+", "char")
    #char_value = String

class Circle(Marker):

    glyphtype = String("circle")

    radius = DataSpec(units="screen", default=4)

#class Rects(Glyph):
#    glyphtype = "rects"
#    x = DataSpec
#    y = DataSpec
#    width = DataSpec
#    height = DataSpec
#    angle = DataSpec
#    color = Color
#    outline_color = Color
#    outline_width = Size

class Quad(Glyph):
    glyphtype = "quad"
    left = DataSpec
    right = DataSpec
    bottom = DataSpec
    top = DataSpec
    #angle = DataSpec

<commit_msg>adding a bunch of placeholder glyph types that correspond to GGPlot; also removing LineRenderer since it no longer exists in BokehJS, and replacing it with the Line glyph type<commit_after>
import inspect

from .properties import (BaseProperty, HasProps, Instance, Enum, Float, Int,
        Color, Percent, Size, Bool, Pattern, Align, Angle, String)

from .objects import PlotObject, Viewable

# Size is a way to preserve a data-space-related metric all the way until
#   render time, when the screen dimensions are known
# Align may be "center", "min", "max", or "jitter(func)" where func is name
#   of a random distribution to draw random samples from. Defaults to uniform
#   but gaussian could certainly be useful.

#class Scene(object):
#    """ A set of related plots, each with their own panel(s) and each
#    panel representing some coherent coordinate space onto which Glyphs
#    are positioned and rendered.
#    """

class DataSpec(BaseProperty):
    """ Because the BokehJS glyphs support a fixed value or a named
    field for most data fields, we capture that in this property.

    We mirror the JS convention in this Python descriptor.  There are
    multiple ways to set a DataSpec, illustrated below with comments
    and example code.

    class Foo(HasProps):
        x = DataSpec("x", units="data")

    f = Foo()
    f.x = "fieldname"   # Sets x to a field named "fieldname"
    f.x = 12            # Sets x to a default value of 12
    f.x = {"name": "foo", "default": 12}

    Just reading out "f.x" will always retrieve what has been set.
    """
    # TODO: Check to see if the "units" field hasn't been removed...

    def __init__(self, field=None, units="data", default=None):
        """
        Parameters
        ==========
        **field** is the string name of a data column to look up.
        **units** is either "data" or "screen"
        **default** is the default value to use if a datapoint is
        missing the field specified in **name**

        If a constant value is desired, then leave **field** as None and
        set **default** to the fixed value.
        """
        # Don't use .name because the HasProps metaclass uses that to
        # store the attribute name on this descriptor.
        self.field = field
        self.units = units
        self.default = default

    @classmethod
    def autocreate(cls, name=None):
        # In this case, use the name the user assigned this DataSpec to
        # as the default field name.
        d = cls(field=name)
        return d

    def __get__(self, obj, cls=None):
        # It's kind of an open question what we should return here if the
        # user hasn't set anything yet.  We use our best heuristic and
        # return a name, a default, or a dict with both, depending on
        # what is None.
        if hasattr(obj, "_"+self.name):
            return getattr(obj, "_"+self.name)
        else:
            if self.field is None:
                return self.default
            elif self.default is None:
                return self.field
            else:
                return {"field": self.field, "default": self.default}

    def __set__(self, obj, value):
        setattr(obj, "_"+self.name, value)

    def __delete__(self, obj):
        if hasattr(obj, self.name + "_dict"):
            delattr(obj, self.name + "_dict")
        BaseProperty.__delete__(self, obj)

    def to_dict(self, obj):
        # Build the complete dict
        value = getattr(obj, "_"+self.name, self.default)
        if type(value) == str:
            d = {"name": value, "units": self.units, "default": self.default}
        elif isinstance(value, dict):
            d = {"name": self.field, "units": self.units, "default": self.default}
            d.update(value)
        else:
            # Assume value is a numeric type and is the default value.
            # We explicitly set the field name to None.
            d = {"name": None, "units": self.units, "default": value}
        return d

class FillProps(HasProps):
    """ Mirrors the BokehJS properties.fill_properties class """
    fill = Color("gray")
    fill_alpha = Percent(1.0)

class LineProps(HasProps):
    """ Mirrors the BokehJS properties.line_properties class """
    line_color = Color("red")
    line_width = Size(1)
    line_alpha = Percent(1.0)
    line_join = String("miter")
    line_cap = String("butt")
    line_dash = Pattern
    line_dash_offset = Int(0)

class TextProps(HasProps):
    """ Mirrors the BokehJS properties.text_properties class """
    text_font = String
    text_font_size = Int(10)
    text_font_style = Enum("normal", "italic", "bold")
    text_color = Color("black")
    text_alpha = Percent(1.0)
    text_align = Enum("left", "right", "center")
    text_baseline = Enum("top", "middle", "bottom")

class MetaGlyph(Viewable):
    """ Handles DataSpecs and other special attribute processing that Glyph
    subclasses need.

    I contemplated making this a class decorator, or monkeypatching the
    Viewable metaclass, but that all seemed more brittle than just
    making a sub-metaclass.
    """

    def __new__(cls, class_name, bases, class_dict):
        # This is a little unorthodox. Usually metaclasses munge the class_dict
        # and attributes before handing off to the base metaclass ctor. In our
        # case, we want the MetaHasProps.__new__ to handle all the machinery
        # of autocreating Property declarations and whatnot, and so we just
        # muck with the class after it's been created.  I've not seen it done
        # this way elsewhere, but it shouldn't be a problem.

        newcls = super(MetaGlyph,cls).__new__(cls, class_name, bases, class_dict)
        dataspecs = {}
        for attr,val in newcls.__dict__.iteritems():
            if isinstance(val, DataSpec):
                dataspecs[attr] = val
        setattr(newcls, "_dataspecs", dataspecs)
        return newcls

class Glyph(PlotObject):
    """ Base class for all glyphs/marks/geoms/whatever-you-call-'em in Bokeh.
    """

    __metaclass__ = MetaGlyph

    # The name that will be used for the 'type' field of the glyph when we
    # create the JS dict representation of this glyph.  "glyph" is not a valid
    # glyph type; it's just a placeholder.
    glyphtype = "glyph"

    # Common attributes for all glyphs
    #color = Color
    #alpha = Percent
    visible = Bool
    margin = Size   # the amount of desired space around this glyph
    halign = Align  # when there is horizontal wiggle room (e.g. categorical)
    valign = Align  # when there is vertical wiggle room

    radius_units = Enum("screen", "data")
    length_units = Enum("screen", "data")
    angle_units = Enum("deg", "rad")
    start_angle_units = Enum("deg", "rad")
    end_angle_units = Enum("deg", "rad")

    def to_glyphspec(self):
        """ Returns a dict mapping attributes to values, that is amenable for
        inclusion in a GlyphRenderer definition.
        """
        d = self.vm_props(withvalues=True)
        d["type"] = d.pop("glyphtype")
        # Iterate over all the DataSpec properties and convert them, using the
        # fact that DataSpecs store the dict-ified version on the object.
        for attrname, dspec in self.dataspecs().iteritems():
            d[attrname] = dspec.to_dict(self)
        return d

    def dataspecs(self):
        dataspecs = {}
        for cls in reversed(inspect.getmro(self.__class__)):
            if isinstance(cls, MetaGlyph):
                dataspecs.update(cls._dataspecs)
        return dataspecs

class Marker(Glyph, FillProps, LineProps):
    """ Base class for glyphs which are just simple markers placed at (x,y)
    locations.
    """

    x = DataSpec
    y = DataSpec
    #size = Size

    # TODO: Remove this and use the color attributes in Glyph instead
    #stroke_color = Color
    #fill_pattern = Pattern
    #shape = Enum("circle", "dot", "square", "tri", "diamond", "x", "+", "char")
    #char_value = String

class Circle(Marker):

    glyphtype = String("circle")

    radius = DataSpec(units="screen", default=4)

#class Rects(Glyph):
#    glyphtype = "rects"
#    x = DataSpec
#    y = DataSpec
#    width = DataSpec
#    height = DataSpec
#    angle = DataSpec
#    color = Color
#    outline_color = Color
#    outline_width = Size

# Other kinds of Markers, to match what GGplot provides
class Triangle(Marker):
    glyphtype = String("triangle")

class Cross(Marker):
    glyphtype = String("cross")

class Xmarker(Marker):
    glyphtype = String("xmarker")

class Diamond(Marker):
    glyphtype = String("diamond")

class InvertedTriangle(Marker):
    glyphtype = String("invertedtriangle")

class SquareX(Marker):
    glyphtype = String("squarex")

class Asterisk(Marker):
    glyphtype = String("asterisk")

class DiamondCross(Marker):
    glyphtype = String("diamondcross")

class CircleCross(Marker):
    glyphtype = String("circlecross")

class HexStar(Marker):
    glyphtype = String("hexstar")

class SquareCross(Marker):
    glyphtype = String("squarecross")

class CircleX(Marker):
    glyphtype = String("circlex")

class Quad(Glyph):
    glyphtype = String("quad")
    left = DataSpec
    right = DataSpec
    bottom = DataSpec
    top = DataSpec
    #angle = DataSpec

class Line(Glyph, LineProps):
    glyphtype = String("line")
    xs = DataSpec
    ys = DataSpec

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import getpass
import logging
from openquake.baselib import sap, config, datastore
from openquake.baselib.general import safeprint
from openquake.hazardlib import valid
from openquake.commonlib import logs, readinput
from openquake.engine import engine as eng
from openquake.engine.export import core
from openquake.engine.utils import confirm
from openquake.engine.tools.make_html_report import make_report
from openquake.server import dbserver
from openquake.commands.abort import abort

HAZARD_CALCULATION_ARG = "--hazard-calculation-id"
MISSING_HAZARD_MSG = "Please specify '%s=<id>'" % HAZARD_CALCULATION_ARG

def read(calc_id, username=None):
    """
    :param calc_id: a calculation ID
    :returns: the associated DataStore instance
    """
    if calc_id == -1 and not username:
        # make sure you get the last calculation of the current user
        username = getpass.getuser()
    job = logs.dbcmd('get_job', calc_id, username)
    if job:
        return datastore.read(job.ds_calc_dir + '.hdf5')
    # calc_id can be present in the datastore and not in the database:
    # this happens if the calculation was run with `oq run`
    return datastore.read(calc_id)

def get_job_id(job_id, username=None):
    job = logs.dbcmd('get_job', job_id, username)
    if not job:
        sys.exit('Job %s not found' % job_id)
    return job.id

def run_job(job_ini, log_level='info', log_file=None, exports='',
            username=getpass.getuser(), **kw):
    """
    Run a job using the specified config file and other options.

    :param str job_ini:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param username:
        Name of the user running the job
    :param kw:
        Extra parameters like hazard_calculation_id and calculation_mode
    """
    job_id = logs.init('job', getattr(logging, log_level.upper()))
    with logs.handle(job_id, log_level, log_file):
        job_ini = os.path.abspath(job_ini)
        oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
        kw['username'] = username
        eng.run_calc(job_id, oqparam, exports, **kw)
        for line in logs.dbcmd('list_outputs', job_id, False):
            safeprint(line)
    return job_id

def run_tile(job_ini, sites_slice):
    """
    Used in tiling calculations
    """
    return run_job(job_ini, sites_slice=(sites_slice.start, sites_slice.stop))

def del_calculation(job_id, confirmed=False):
    """
    Delete a calculation and all associated outputs.
    """
    if logs.dbcmd('get_job', job_id) is None:
        print('There is no job %d' % job_id)
        return

    if confirmed or confirm(
            'Are you sure you want to (abort and) delete this calculation and '
            'all associated outputs?\nThis action cannot be undone. (y/n): '):
        try:
            abort.func(job_id)
            resp = logs.dbcmd('del_calc', job_id, getpass.getuser())
        except RuntimeError as err:
            safeprint(err)
        else:
            if 'success' in resp:
                print('Removed %d' % job_id)
            else:
                print(resp['error'])

class EBRunner(object):
    """
    A class to run event based risk calculations
    """
    def __init__(self, job_ini, oqparam, log_level, log_file, exports,
                 reuse_hazard):
        self.job_ini = job_ini
        self.oqparam = oqparam
        self.log_level = log_level
        self.log_file = log_file
        self.exports = exports
        checksum = readinput.get_hazard_checksum32(oqparam)
        # retrieve an old calculation with the right checksum, if any
        job = logs.dbcmd('get_job_from_checksum', checksum)
        kw = dict(calculation_mode='event_based')
        if (oqparam.sites or 'sites' in oqparam.inputs or
                'site_model' in oqparam.inputs):
            # remove exposure from the hazard
            kw['exposure_file'] = ''
        if job is None:
            # recompute the hazard and store the checksum
            self.hc_id = run_job(job_ini, log_level, log_file, exports, **kw)
            logs.dbcmd('add_checksum', self.hc_id, checksum)
        elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'):
            # recompute and update the job associated to the checksum
            self.hc_id = run_job(job_ini, log_level, log_file, exports, **kw)
            logs.dbcmd('update_job_checksum', self.hc_id, checksum)
        else:
            # sanity check
            assert job.description == oqparam.description, (
                job.description, oqparam.description)
            self.hc_id = job.id
            logging.info('Reusing job #%d', job.id)

    def run_risk(self):
        run_job(self.job_ini, self.log_level, self.log_file,
                self.exports, hazard_calculation_id=self.hc_id)

@sap.Script
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, db_version, what_if_I_upgrade, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info', reuse_hazard=False):
    """
    Run a calculation using the traditional command line API
    """
    if not run:
        # configure a basic logging
        logs.init()

    if config_file:
        config.read(os.path.abspath(os.path.expanduser(config_file)),
                    soft_mem_limit=int, hard_mem_limit=int, port=int,
                    multi_user=valid.boolean, multi_node=valid.boolean)

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id == -1:
        # get the latest calculation of the current user
        hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
    elif hazard_calculation_id:
        # make it possible to use calculations made by another user
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        job_inis = [os.path.expanduser(f) for f in run]
        if len(job_inis) == 1 and not hc_id:
            # init logs before calling get_oqparam but without creating a job
            logs.init('nojob', getattr(logging, log_level.upper()))
            # not using logs.handle that logs on the db
            oq = readinput.get_oqparam(job_inis[0])
            if oq.calculation_mode.startswith('event_based'):
                ebr = EBRunner(job_inis[0], oq, log_level, log_file,
                               exports, reuse_hazard)
                if oq.calculation_mode.endswith('risk'):
                    ebr.run_risk()
                return
        for i, job_ini in enumerate(job_inis):
            open(job_ini, 'rb').read()  # IOError if the file does not exist
            job_id = run_job(job_ini, log_level, log_file,
                             exports, hazard_calculation_id=hc_id)
            if not hc_id:  # use the first calculation as base for the others
                hc_id = job_id
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            safeprint(line)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'csv,xml'):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()

engine._add('log_file', '--log-file', '-L', help='''\
Location where to store log messages; if not specified, log messages
will be printed to the console (to stderr)''')
engine._add('no_distribute', '--no-distribute', '--nd', help='''\
Disable calculation task distribution and run the
computation in a single process. This is intended for
use in debugging and profiling.''', action='store_true')
engine.flg('yes', 'Automatically answer "yes" when asked to confirm an action')
engine.opt('config_file', 'Custom openquake.cfg file, to override default '
           'configurations')
engine._add('make_html_report', '--make-html-report', '--r',
            help='Build an HTML report of the computation at the given date',
            metavar='YYYY-MM-DD|today')
engine.flg('upgrade_db', 'Upgrade the openquake database')
engine.flg('db_version', 'Show the current version of the openquake database')
engine.flg('what_if_I_upgrade', 'Show what will happen to the openquake '
           'database if you upgrade')
engine._add('run', '--run', help='Run a job with the specified config file',
            metavar='JOB_INI', nargs='+')
engine._add('list_hazard_calculations', '--list-hazard-calculations', '--lhc',
            help='List hazard calculation information', action='store_true')
engine._add('list_risk_calculations', '--list-risk-calculations', '--lrc',
            help='List risk calculation information', action='store_true')
engine._add('delete_calculation', '--delete-calculation', '--dc',
            help='Delete a calculation and all associated outputs',
            metavar='CALCULATION_ID', type=int)
engine._add('delete_uncompleted_calculations',
            '--delete-uncompleted-calculations', '--duc',
            help='Delete all the uncompleted calculations',
            action='store_true')
engine._add('hazard_calculation_id', '--hazard-calculation-id', '--hc',
            help='Use the given job as input for the next job')
engine._add('list_outputs', '--list-outputs', '--lo',
            help='List outputs for the specified calculation',
            metavar='CALCULATION_ID')
engine._add('show_log', '--show-log', '--sl',
            help='Show the log of the specified calculation',
            metavar='CALCULATION_ID')
engine._add('export_output', '--export-output', '--eo',
            nargs=2, metavar=('OUTPUT_ID', 'TARGET_DIR'),
            help='Export the desired output to the specified directory')
engine._add('export_outputs', '--export-outputs', '--eos',
            nargs=2, metavar=('CALCULATION_ID', 'TARGET_DIR'),
            help='Export all of the calculation outputs to the '
            'specified directory')
engine.opt('exports', 'Comma-separated string specifing the export formats, '
           'in order of priority')
engine.opt('log_level', 'Defaults to "info"',
           choices=['debug', 'info', 'warn', 'error', 'critical'])
engine.flg('reuse_hazard', 'Reuse the event based hazard if available')
<commit_msg>Fixed docstring [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import getpass
import logging
from openquake.baselib import sap, config, datastore
from openquake.baselib.general import safeprint
from openquake.hazardlib import valid
from openquake.commonlib import logs, readinput
from openquake.engine import engine as eng
from openquake.engine.export import core
from openquake.engine.utils import confirm
from openquake.engine.tools.make_html_report import make_report
from openquake.server import dbserver
from openquake.commands.abort import abort

HAZARD_CALCULATION_ARG = "--hazard-calculation-id"
MISSING_HAZARD_MSG = "Please specify '%s=<id>'" % HAZARD_CALCULATION_ARG

def read(calc_id, username=None):
    """
    :param calc_id: a calculation ID
    :param username: if given, restrict the search to the user's calculations
    :returns: the associated DataStore instance
    """
    if calc_id == -1 and not username:
        # make sure you get the last calculation of the current user
        username = getpass.getuser()
    job = logs.dbcmd('get_job', calc_id, username)
    if job:
        return datastore.read(job.ds_calc_dir + '.hdf5')
    # calc_id can be present in the datastore and not in the database:
    # this happens if the calculation was run with `oq run`
    return datastore.read(calc_id)

def get_job_id(job_id, username=None):
    job = logs.dbcmd('get_job', job_id, username)
    if not job:
        sys.exit('Job %s not found' % job_id)
    return job.id

def run_job(job_ini, log_level='info', log_file=None, exports='',
            username=getpass.getuser(), **kw):
    """
    Run a job using the specified config file and other options.

    :param str job_ini:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param username:
        Name of the user running the job
    :param kw:
        Extra parameters like hazard_calculation_id and calculation_mode
    """
    job_id = logs.init('job', getattr(logging, log_level.upper()))
    with logs.handle(job_id, log_level, log_file):
        job_ini = os.path.abspath(job_ini)
        oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
        kw['username'] = username
        eng.run_calc(job_id, oqparam, exports, **kw)
        for line in logs.dbcmd('list_outputs', job_id, False):
            safeprint(line)
    return job_id

def run_tile(job_ini, sites_slice):
    """
    Used in tiling calculations
    """
    return run_job(job_ini, sites_slice=(sites_slice.start, sites_slice.stop))

def del_calculation(job_id, confirmed=False):
    """
    Delete a calculation and all associated outputs.
    """
    if logs.dbcmd('get_job', job_id) is None:
        print('There is no job %d' % job_id)
        return

    if confirmed or confirm(
            'Are you sure you want to (abort and) delete this calculation and '
            'all associated outputs?\nThis action cannot be undone. (y/n): '):
        try:
            abort.func(job_id)
            resp = logs.dbcmd('del_calc', job_id, getpass.getuser())
        except RuntimeError as err:
            safeprint(err)
        else:
            if 'success' in resp:
                print('Removed %d' % job_id)
            else:
                print(resp['error'])

class EBRunner(object):
    """
    A class to run event based risk calculations
    """
    def __init__(self, job_ini, oqparam, log_level, log_file, exports,
                 reuse_hazard):
        self.job_ini = job_ini
        self.oqparam = oqparam
        self.log_level = log_level
        self.log_file = log_file
        self.exports = exports
        checksum = readinput.get_hazard_checksum32(oqparam)
        # retrieve an old calculation with the right checksum, if any
        job = logs.dbcmd('get_job_from_checksum', checksum)
        kw = dict(calculation_mode='event_based')
        if (oqparam.sites or 'sites' in oqparam.inputs or
                'site_model' in oqparam.inputs):
            # remove exposure from the hazard
            kw['exposure_file'] = ''
        if job is None:
            # recompute the hazard and store the checksum
            self.hc_id = run_job(job_ini, log_level, log_file, exports, **kw)
            logs.dbcmd('add_checksum', self.hc_id, checksum)
        elif not reuse_hazard or not os.path.exists(job.ds_calc_dir + '.hdf5'):
            # recompute and update the job associated to the checksum
            self.hc_id = run_job(job_ini, log_level, log_file, exports, **kw)
            logs.dbcmd('update_job_checksum', self.hc_id, checksum)
        else:
            # sanity check
            assert job.description == oqparam.description, (
                job.description, oqparam.description)
            self.hc_id = job.id
            logging.info('Reusing job #%d', job.id)

    def run_risk(self):
        run_job(self.job_ini, self.log_level, self.log_file,
                self.exports, hazard_calculation_id=self.hc_id)

@sap.Script
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, db_version, what_if_I_upgrade, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info', reuse_hazard=False):
    """
    Run a calculation using the traditional command line API
    """
    if not run:
        # configure a basic logging
        logs.init()

    if config_file:
        config.read(os.path.abspath(os.path.expanduser(config_file)),
                    soft_mem_limit=int, hard_mem_limit=int, port=int,
                    multi_user=valid.boolean, multi_node=valid.boolean)

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id == -1:
        # get the latest calculation of the current user
        hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
    elif hazard_calculation_id:
        # make it possible to use calculations made by another user
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        job_inis = [os.path.expanduser(f) for f in run]
        if len(job_inis) == 1 and not hc_id:
            # init logs before calling get_oqparam but without creating a job
            logs.init('nojob', getattr(logging, log_level.upper()))
            # not using logs.handle that logs on the db
            oq = readinput.get_oqparam(job_inis[0])
            if oq.calculation_mode.startswith('event_based'):
                ebr = EBRunner(job_inis[0], oq, log_level, log_file,
                               exports, reuse_hazard)
                if oq.calculation_mode.endswith('risk'):
                    ebr.run_risk()
                return
        for i, job_ini in enumerate(job_inis):
            open(job_ini, 'rb').read()  # IOError if the file does not exist
            job_id = run_job(job_ini, log_level, log_file,
                             exports, hazard_calculation_id=hc_id)
            if not hc_id:  # use the first calculation as base for the others
                hc_id = job_id
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            safeprint(line)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'csv,xml'):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()

engine._add('log_file', '--log-file', '-L', help='''\
Location where to store log messages; if not specified, log messages
will be printed to the console (to stderr)''')
engine._add('no_distribute', '--no-distribute', '--nd', help='''\
Disable calculation task distribution and run the
computation in a single process. This is intended for
use in debugging and profiling.''', action='store_true')
engine.flg('yes', 'Automatically answer "yes" when asked to confirm an action')
engine.opt('config_file', 'Custom openquake.cfg file, to override default '
           'configurations')
engine._add('make_html_report', '--make-html-report', '--r',
            help='Build an HTML report of the computation at the given date',
            metavar='YYYY-MM-DD|today')
engine.flg('upgrade_db', 'Upgrade the openquake database')
engine.flg('db_version', 'Show the current version of the openquake database')
engine.flg('what_if_I_upgrade', 'Show what will happen to the openquake '
           'database if you upgrade')
engine._add('run', '--run', help='Run a job with the specified config file',
            metavar='JOB_INI', nargs='+')
engine._add('list_hazard_calculations', '--list-hazard-calculations', '--lhc',
            help='List hazard calculation information', action='store_true')
engine._add('list_risk_calculations', '--list-risk-calculations', '--lrc',
            help='List risk calculation information', action='store_true')
engine._add('delete_calculation', '--delete-calculation', '--dc',
            help='Delete a calculation and all associated outputs',
            metavar='CALCULATION_ID', type=int)
engine._add('delete_uncompleted_calculations',
            '--delete-uncompleted-calculations', '--duc',
            help='Delete all the uncompleted calculations',
            action='store_true')
engine._add('hazard_calculation_id', '--hazard-calculation-id', '--hc',
            help='Use the given job as input for the next job')
engine._add('list_outputs', '--list-outputs', '--lo',
            help='List outputs for the specified calculation',
            metavar='CALCULATION_ID')
engine._add('show_log', '--show-log', '--sl',
            help='Show the log of the specified calculation',
            metavar='CALCULATION_ID')
engine._add('export_output', '--export-output', '--eo',
            nargs=2, metavar=('OUTPUT_ID', 'TARGET_DIR'),
            help='Export the desired output to the specified directory')
engine._add('export_outputs', '--export-outputs', '--eos',
            nargs=2, metavar=('CALCULATION_ID', 'TARGET_DIR'),
            help='Export all of the calculation outputs to the '
            'specified directory')
engine.opt('exports', 'Comma-separated string specifing the export formats, '
           'in order of priority')
engine.opt('log_level', 'Defaults to "info"',
           choices=['debug', 'info', 'warn', 'error', 'critical'])
engine.flg('reuse_hazard', 'Reuse the event based hazard if available')
<|endoftext|>


<commit_before># Copyright 2019 Akretion
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Spec Driven Model',
    'summary': """
        Tools for specifications driven mixins (from xsd for instance)""",
    'version': '12.0.1.0.0',
    'maintainers': ['rvalyi'],
    'license': 'LGPL-3',
    'author': 'Akretion,Odoo Community Association (OCA)',
    'depends': [
    ],
    'data': [
    ],
    'demo': [
    ],
    'development_status': 'Beta',
}
<commit_msg>spec_driven_model 12.0.1.1.0<commit_after># Copyright 2019 Akretion
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Spec Driven Model',
    'summary': """
        Tools for specifications driven mixins (from xsd for instance)""",
    'version': '12.0.1.1.0',
    'maintainers': ['rvalyi'],
    'license': 'LGPL-3',
    'author': 'Akretion,Odoo Community Association (OCA)',
    'depends': [
    ],
    'data': [
    ],
    'demo': [
    ],
    'development_status': 'Beta',
}
<|endoftext|>


<commit_before>"""
============
bootstrapper
============

Bootstrap Python projects or libraries with ease.

"""

from __future__ import print_function

import copy
import operator
import os
import platform
import subprocess
import sys
import tempfile
import traceback

try:
    from configparser import Error as ConfigParserError, SafeConfigParser
except ImportError:
    from ConfigParser import Error as ConfigParserError, SafeConfigParser

from collections import defaultdict
from contextlib import contextmanager
from distutils.util import strtobool
from functools import wraps

try:
    from pip.log import _color_wrap
    from pip._vendor import colorama
except ImportError:
    colorama = None

__author__ = 'Igor Davydenko'
__license__ = 'BSD License'
__script__ = 'bootstrapper'
__version__ = '0.3-dev'

IS_PY3 = sys.version_info[0] == 3
IS_WINDOWS = platform.system() == 'Windows'

iteritems = lambda seq: seq.items() if IS_PY3 else seq.iteritems()
iterkeys = lambda seq: seq.keys() if IS_PY3 else seq.iterkeys()
safe_path = lambda value: value.replace('/', os.sep) if IS_WINDOWS else value
string_types = (bytes, str) if IS_PY3 else (basestring, )  # noqa

BOOTSTRAPPER_TEST_KEY = 'BOOTSTRAPPER_TEST'
CONFIG = {
    __script__: {
        'env': 'env',
        'requirements': 'requirements.txt',
        'quiet': False,
    },
    'pip': {
        'download_cache': safe_path(os.path.expanduser(
            os.path.join('~', '.{0}'.format(__script__), 'pip-cache')
        )),
    },
    'virtualenv': {},
}
DEFAULT_CONFIG = 'bootstrap.cfg'
ERROR_HANDLER_DISABLED = False

def check_pre_requirements(pre_requirements):
    """
    Check all necessary system requirements to exist.
    """
    pre_requirements = set(pre_requirements or [])
    pre_requirements.add('virtualenv')

    for requirement in pre_requirements:
        if not which(requirement):
            print_error('Requirement {0!r} is not found in system'.
                        format(requirement))
            return False

    return True

def config_to_args(config):
    """
    Convert config dict to arguments list.
    """
    result = []

    for key, value in iteritems(config):
        if value is False:
            continue

        key = '--{0}'.format(key.replace('_', '-'))

        if isinstance(value, (list, set, tuple)):
            for item in value:
                result.extend((key, smart_str(item)))
        elif value is not True:
            result.extend((key, smart_str(value)))
        else:
            result.append(key)

    return tuple(result)

def create_env(env, args, recreate=False, ignore_activated=False, quiet=False):
    """
    Create virtual environment.
    """
    cmd = None

    inside_env = hasattr(sys, 'real_prefix') or os.environ.get('VIRTUAL_ENV')
    env_exists = os.path.isdir(env)

    if not quiet:
        print('== Step 1. Create virtual environment ==')

    if (
        recreate or (not inside_env and not env_exists)
    ) or (
        ignore_activated and not env_exists
    ):
        cmd = ('virtualenv', ) + args + (env, )

    if not cmd and not quiet:
        if inside_env:
            message = 'Working inside of virtual environment, done...'
        else:
            message = 'Virtual environment {0!r} already created, done...'
        print(message.format(env))

    if cmd:
        with disable_error_handler():
            result = not run_cmd(cmd, echo=not quiet)

    if not quiet:
        print()

    return result

@contextmanager
def disable_error_handler():
    """
    Temporary disable error handling.
    """
    global ERROR_HANDLER_DISABLED
    ERROR_HANDLER_DISABLED = True
    yield
    ERROR_HANDLER_DISABLED = False

def error_handler(func):
    """
    Decorator to  error handling.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        """
        Run actual function and if exception catched and error handler enabled
        put traceback to log file
        """
        try:
            return func(*args, **kwargs)
        except BaseException as err:
            # Do not catch exceptions on testing
            if BOOTSTRAPPER_TEST_KEY in os.environ:
                raise
            # Fail silently if error handling disabled
            if ERROR_HANDLER_DISABLED:
                return True
            # Otherwise save traceback to log
            return save_traceback(err)
    return wrapper

def get_temp_streams():
    """
    Return two temporary file handlers for STDOUT and STDERR.
    """
    kwargs = {'encoding': 'utf-8'} if IS_PY3 else {}
    return (tempfile.TemporaryFile('w+', **kwargs),
            tempfile.TemporaryFile('w+', **kwargs))

def install(env, requirements, args, ignore_activated=False, quiet=False):
    """
    Install library or project into virtual environment.
    """
    if os.path.isfile(requirements):
        args += ('-r', requirements)
        label = 'project'
    else:
        args += ('-U', '-e', '.')
        label = 'library'

    if not quiet:
        print('== Step 2. Install {0} =='.format(label))

    pip_cmd(env, ('install', ) + args, ignore_activated, echo=not quiet)

    if not quiet:
        print()

    return True

@error_handler
def main(*args):
    """
    Bootstrap Python projects and libraries with virtualenv and pip.

    Also check system requirements before bootstrap and run post bootstrap
    hook if any.
    """
    # Create parser, read arguments from direct input or command line
    args = parse_args(args or sys.argv[1:])

    # Read current config from file and command line arguments
    config = read_config(args.config, args)
    if config is None:
        return True
    bootstrap = config[__script__]

    # Check pre-requirements
    if not check_pre_requirements(bootstrap['pre_requirements']):
        return True

    # Create virtual environment
    env_args = prepare_args(config['virtualenv'], bootstrap)
    create_env(bootstrap['env'],
               env_args,
               bootstrap['recreate'],
               bootstrap['ignore_activated'],
               bootstrap['quiet'])

    # And install library or project here
    pip_args = prepare_args(config['pip'], bootstrap)
    install(bootstrap['env'],
            bootstrap['requirements'],
            pip_args,
            bootstrap['ignore_activated'],
            bootstrap['quiet'])

    # Run post-bootstrap hook
    run_hook(bootstrap['hook'], bootstrap, bootstrap['quiet'])

    # All OK!
    if not bootstrap['quiet']:
        print('All OK!')

    # False means everything went alright, exit code: 0
    return False

def parse_args(args):
    """
    Parse args from command line by creating argument parser instance and
    process it.
    """
    from argparse import ArgumentParser

    description = ('Bootstrap Python projects and libraries with virtualenv '
                   'and pip.')
    parser = ArgumentParser(description=description)
    parser.add_argument('--version', action='version', version=__version__)

    parser.add_argument(
        '-c', '--config', default=DEFAULT_CONFIG,
        help='Path to config file. By default: {0}'.format(DEFAULT_CONFIG)
    )
    parser.add_argument(
        '-p', '--pre-requirements', default=[], nargs='+',
        help='List of pre-requirements to check, separated by space.'
    )
    parser.add_argument(
        '-e', '--env',
        help='Virtual environment name. By default: {0}'.
             format(CONFIG[__script__]['env'])
    )
    parser.add_argument(
        '-r', '--requirements',
        help='Path to requirements file. By default: {0}'.
             format(CONFIG[__script__]['requirements']))
    parser.add_argument(
        '-C', '--hook', help='Execute this hook after bootstrap process.'
    )
    parser.add_argument(
        '--ignore-activated', action='store_true', default=None,
        help='Ignore pre-activated virtualenv, like on Travis CI.'
    )
    parser.add_argument(
        '--recreate', action='store_true', default=None,
        help='Recreate virtualenv on every run.'
    )
    parser.add_argument(
        '-q', '--quiet', action='store_true', default=None,
        help='Minimize output, show only error messages.'
    )

    return parser.parse_args(args)

def pip_cmd(env, cmd, ignore_activated=False, **kwargs):
    """
    Run pip command in given or activated virtual environment.
    """
    cmd = tuple(cmd)
    dirname = safe_path(env)

    if not ignore_activated:
        activated_env = os.environ.get('VIRTUAL_ENV')

        if hasattr(sys, 'real_prefix'):
            dirname = sys.prefix
        elif activated_env:
            dirname = activated_env

    pip_path = os.path.join(dirname, 'Scripts' if IS_WINDOWS else 'bin', 'pip')

    if kwargs.pop('return_path', False):
        return pip_path

    if not os.path.isfile(pip_path):
        raise OSError('No pip found at {0!r}'.format(pip_path))

    with disable_error_handler():
        return not run_cmd((pip_path, ) + cmd, **kwargs)

def prepare_args(config, bootstrap):
    """
    Convert config dict to command line args line.
    """
    config = copy.deepcopy(config)
    environ = dict(copy.deepcopy(os.environ))

    data = {'env': bootstrap['env'],
            'pip': pip_cmd(bootstrap['env'], '', return_path=True),
            'requirements': bootstrap['requirements']}
    environ.update(data)

    if isinstance(config, string_types):
        return config.format(**environ)

    for key, value in iteritems(config):
        if not isinstance(value, string_types):
            continue
        config[key] = value.format(**environ)

    return config_to_args(config)

def print_error(message, wrap=True):
    """
    Print error message to stderr, using ANSI-colors.
    """
    if wrap:
        message = 'ERROR: {0}. Exit...'.format(message.rstrip('.'))

    colorizer = (_color_wrap(colorama.Fore.RED)
                 if colorama
                 else lambda message: message)
    return print(colorizer(message), file=sys.stderr)

def read_config(filename, args):
    """
    Read and parse configuration file. By default, ``filename`` is relative
    path to current work directory.

    If no config file found, default ``CONFIG`` would be used.
    """
    # Initial vars
    config = defaultdict(dict)
    splitter = operator.methodcaller('split', ' ')

    converters = {
        __script__: {
            'env': safe_path,
            'pre_requirements': splitter,
        },
        'pip': {
            'allow_external': splitter,
            'allow_unverified': splitter,
        }
    }
    default = copy.deepcopy(CONFIG)
    sections = set(iterkeys(default))

    # Expand user and environ vars in config filename
    filename = os.path.expandvars(os.path.expanduser(filename))

    # Read config if it exists on disk
    if os.path.isfile(filename):
        parser = SafeConfigParser()

        try:
            parser.read(filename)
        except ConfigParserError:
            print_error('Cannot parse config file at {0!r}'.format(filename))
            return None

        # Apply config for each possible section
        for section in sections:
            if not parser.has_section(section):
                continue

            items = parser.items(section)

            # Make auto convert here for integers and boolean values
            for key, value in items:
                try:
                    value = int(value)
                except (TypeError, ValueError):
                    try:
                        value = bool(strtobool(value))
                    except ValueError:
                        pass

                if section in converters and key in converters[section]:
                    value = converters[section][key](value)

                config[section][key] = value

    # Update config with default values if necessary
    for section, data in iteritems(default):
        if section not in config:
            config[section] = data
        else:
            for key, value in iteritems(data):
                config[section].setdefault(key, value)

    # Update bootstrap config from parsed args
    keys = set((
        'env', 'hook', 'ignore_activated', 'pre_requirements', 'quiet',
        'recreate', 'requirements'
    ))

    for key in keys:
        value = getattr(args, key)
        config[__script__].setdefault(key, value)

        if key == 'pre_requirements' and not value:
            continue

        if value is not None:
            config[__script__][key] = value

    return config

def run_cmd(cmd, echo=False, fail_silently=False, **kwargs):
    """
    Call given tuple or string with ``subprocess.call`` function.

    If ``echo`` show command to call and its output in STDOUT, otherwise hide
    all output.
    """
    if echo:
        cmd_str = cmd if isinstance(cmd, string_types) else ' '.join(cmd)
        kwargs['stdout'], kwargs['stderr'] = sys.stdout, sys.stderr
        print('$ {0}'.format(cmd_str))
    else:
        out, err = get_temp_streams()
        kwargs['stdout'], kwargs['stderr'] = out, err

    try:
        retcode = subprocess.call(cmd, **kwargs)
    except subprocess.CalledProcessError as err:
        if fail_silently:
            return False
        print_error(str(err) if IS_PY3 else unicode(err))  # noqa

    if retcode and echo and not fail_silently:
        print_error('Command {0!r} returned non-zero exit status {1}'.
                    format(cmd_str, retcode))

    return retcode

def run_hook(hook, config, quiet=False):
    """
    Run post-bootstrap hook if any.
    """
    if not hook:
        return True

    if not quiet:
        print('== Step 3. Run post-bootstrap hook ==')

    result = not run_cmd(prepare_args(hook, config),
                         echo=not quiet,
                         fail_silently=True,
                         shell=True)

    if not quiet:
        print()

    return result

def save_traceback(err):
    """
    Save error traceback to bootstrapper log file.
    """
    # Store logs to ~/.bootstrapper directory
    dirname = safe_path(os.path.expanduser(
        os.path.join('~', '.{0}'.format(__script__))
    ))

    # But ensure that directory exists
    if not os.path.isdir(dirname):
        os.mkdir(dirname)

    # Now we ready to put traceback to log file
    filename = os.path.join(dirname, '{0}.log'.format(__script__))

    with open(filename, 'a+') as handler:
        traceback.print_exc(file=handler)

    # And show colorized message
    message = ('User aborted workflow'
               if isinstance(err, KeyboardInterrupt)
               else 'Unexpected error catched')
    print_error(message)
    print_error('Full log stored to {0}'.format(filename), False)

    return True

def smart_str(value, encoding='utf-8', errors='strict'):
    """
    Convert Python object to string.
    """
    if not IS_PY3 and isinstance(value, unicode):  # noqa
        return value.encode(encoding, errors)
    return str(value)

def which(executable):
    """
    Shortcut to check whether executable available in current environment or
    not.
    """
    cmd = 'where' if IS_WINDOWS else 'which'
    return not run_cmd((cmd, executable), fail_silently=True)

if __name__ == '__main__':
    sys.exit(int(main()))
<commit_msg>Disable handling errors on parsing args from command line.<commit_after>"""
============
bootstrapper
============

Bootstrap Python projects or libraries with ease.

"""

from __future__ import print_function

import copy
import operator
import os
import platform
import subprocess
import sys
import tempfile
import traceback

try:
    from configparser import Error as ConfigParserError, SafeConfigParser
except ImportError:
    from ConfigParser import Error as ConfigParserError, SafeConfigParser

from collections import defaultdict
from contextlib import contextmanager
from distutils.util import strtobool
from functools import wraps

try:
    from pip.log import _color_wrap
    from pip._vendor import colorama
except ImportError:
    colorama = None

__author__ = 'Igor Davydenko'
__license__ = 'BSD License'
__script__ = 'bootstrapper'
__version__ = '0.3-dev'

IS_PY3 = sys.version_info[0] == 3
IS_WINDOWS = platform.system() == 'Windows'

iteritems = lambda seq: seq.items() if IS_PY3 else seq.iteritems()
iterkeys = lambda seq: seq.keys() if IS_PY3 else seq.iterkeys()
safe_path = lambda value: value.replace('/', os.sep) if IS_WINDOWS else value
string_types = (bytes, str) if IS_PY3 else (basestring, )  # noqa

BOOTSTRAPPER_TEST_KEY = 'BOOTSTRAPPER_TEST'
CONFIG = {
    __script__: {
        'env': 'env',
        'requirements': 'requirements.txt',
        'quiet': False,
    },
    'pip': {
        'download_cache': safe_path(os.path.expanduser(
            os.path.join('~', '.{0}'.format(__script__), 'pip-cache')
        )),
    },
    'virtualenv': {},
}
DEFAULT_CONFIG = 'bootstrap.cfg'
ERROR_HANDLER_DISABLED = False

def check_pre_requirements(pre_requirements):
    """
    Check all necessary system requirements to exist.
    """
    pre_requirements = set(pre_requirements or [])
    pre_requirements.add('virtualenv')

    for requirement in pre_requirements:
        if not which(requirement):
            print_error('Requirement {0!r} is not found in system'.
                        format(requirement))
            return False

    return True

def config_to_args(config):
    """
    Convert config dict to arguments list.
    """
    result = []

    for key, value in iteritems(config):
        if value is False:
            continue

        key = '--{0}'.format(key.replace('_', '-'))

        if isinstance(value, (list, set, tuple)):
            for item in value:
                result.extend((key, smart_str(item)))
        elif value is not True:
            result.extend((key, smart_str(value)))
        else:
            result.append(key)

    return tuple(result)

def create_env(env, args, recreate=False, ignore_activated=False, quiet=False):
    """
    Create virtual environment.
    """
    cmd = None

    inside_env = hasattr(sys, 'real_prefix') or os.environ.get('VIRTUAL_ENV')
    env_exists = os.path.isdir(env)

    if not quiet:
        print('== Step 1. Create virtual environment ==')

    if (
        recreate or (not inside_env and not env_exists)
    ) or (
        ignore_activated and not env_exists
    ):
        cmd = ('virtualenv', ) + args + (env, )

    if not cmd and not quiet:
        if inside_env:
            message = 'Working inside of virtual environment, done...'
        else:
            message = 'Virtual environment {0!r} already created, done...'
        print(message.format(env))

    if cmd:
        with disable_error_handler():
            result = not run_cmd(cmd, echo=not quiet)

    if not quiet:
        print()

    return result

@contextmanager
def disable_error_handler():
    """
    Temporary disable error handling.
    """
    global ERROR_HANDLER_DISABLED
    ERROR_HANDLER_DISABLED = True
    yield
    ERROR_HANDLER_DISABLED = False

def error_handler(func):
    """
    Decorator to  error handling.
    """
    @wraps(func)
    def wrapper(*args, **kwargs):
        """
        Run actual function and if exception catched and error handler enabled
        put traceback to log file
        """
        try:
            return func(*args, **kwargs)
        except BaseException as err:
            # Do not catch exceptions on testing
            if BOOTSTRAPPER_TEST_KEY in os.environ:
                raise
            # Fail silently if error handling disabled
            if ERROR_HANDLER_DISABLED:
                return True
            # Otherwise save traceback to log
            return save_traceback(err)
    return wrapper

def get_temp_streams():
    """
    Return two temporary file handlers for STDOUT and STDERR.
    """
    kwargs = {'encoding': 'utf-8'} if IS_PY3 else {}
    return (tempfile.TemporaryFile('w+', **kwargs),
            tempfile.TemporaryFile('w+', **kwargs))

def install(env, requirements, args, ignore_activated=False, quiet=False):
    """
    Install library or project into virtual environment.
    """
    if os.path.isfile(requirements):
        args += ('-r', requirements)
        label = 'project'
    else:
        args += ('-U', '-e', '.')
        label = 'library'

    if not quiet:
        print('== Step 2. Install {0} =='.format(label))

    pip_cmd(env, ('install', ) + args, ignore_activated, echo=not quiet)

    if not quiet:
        print()

    return True

@error_handler
def main(*args):
    """
    Bootstrap Python projects and libraries with virtualenv and pip.

    Also check system requirements before bootstrap and run post bootstrap
    hook if any.
    """
    # Create parser, read arguments from direct input or command line
    with disable_error_handler():
        args = parse_args(args or sys.argv[1:])

    # Read current config from file and command line arguments
    config = read_config(args.config, args)
    if config is None:
        return True
    bootstrap = config[__script__]

    # Check pre-requirements
    if not check_pre_requirements(bootstrap['pre_requirements']):
        return True

    # Create virtual environment
    env_args = prepare_args(config['virtualenv'], bootstrap)
    create_env(bootstrap['env'],
               env_args,
               bootstrap['recreate'],
               bootstrap['ignore_activated'],
               bootstrap['quiet'])

    # And install library or project here
    pip_args = prepare_args(config['pip'], bootstrap)
    install(bootstrap['env'],
            bootstrap['requirements'],
            pip_args,
            bootstrap['ignore_activated'],
            bootstrap['quiet'])

    # Run post-bootstrap hook
    run_hook(bootstrap['hook'], bootstrap, bootstrap['quiet'])

    # All OK!
    if not bootstrap['quiet']:
        print('All OK!')

    # False means everything went alright, exit code: 0
    return False

def parse_args(args):
    """
    Parse args from command line by creating argument parser instance and
    process it.
    """
    from argparse import ArgumentParser

    description = ('Bootstrap Python projects and libraries with virtualenv '
                   'and pip.')
    parser = ArgumentParser(description=description)
    parser.add_argument('--version', action='version', version=__version__)

    parser.add_argument(
        '-c', '--config', default=DEFAULT_CONFIG,
        help='Path to config file. By default: {0}'.format(DEFAULT_CONFIG)
    )
    parser.add_argument(
        '-p', '--pre-requirements', default=[], nargs='+',
        help='List of pre-requirements to check, separated by space.'
    )
    parser.add_argument(
        '-e', '--env',
        help='Virtual environment name. By default: {0}'.
             format(CONFIG[__script__]['env'])
    )
    parser.add_argument(
        '-r', '--requirements',
        help='Path to requirements file. By default: {0}'.
             format(CONFIG[__script__]['requirements']))
    parser.add_argument(
        '-C', '--hook', help='Execute this hook after bootstrap process.'
    )
    parser.add_argument(
        '--ignore-activated', action='store_true', default=None,
        help='Ignore pre-activated virtualenv, like on Travis CI.'
    )
    parser.add_argument(
        '--recreate', action='store_true', default=None,
        help='Recreate virtualenv on every run.'
    )
    parser.add_argument(
        '-q', '--quiet', action='store_true', default=None,
        help='Minimize output, show only error messages.'
    )

    return parser.parse_args(args)

def pip_cmd(env, cmd, ignore_activated=False, **kwargs):
    """
    Run pip command in given or activated virtual environment.
    """
    cmd = tuple(cmd)
    dirname = safe_path(env)

    if not ignore_activated:
        activated_env = os.environ.get('VIRTUAL_ENV')

        if hasattr(sys, 'real_prefix'):
            dirname = sys.prefix
        elif activated_env:
            dirname = activated_env

    pip_path = os.path.join(dirname, 'Scripts' if IS_WINDOWS else 'bin', 'pip')

    if kwargs.pop('return_path', False):
        return pip_path

    if not os.path.isfile(pip_path):
        raise OSError('No pip found at {0!r}'.format(pip_path))

    with disable_error_handler():
        return not run_cmd((pip_path, ) + cmd, **kwargs)

def prepare_args(config, bootstrap):
    """
    Convert config dict to command line args line.
    """
    config = copy.deepcopy(config)
    environ = dict(copy.deepcopy(os.environ))

    data = {'env': bootstrap['env'],
            'pip': pip_cmd(bootstrap['env'], '', return_path=True),
            'requirements': bootstrap['requirements']}
    environ.update(data)

    if isinstance(config, string_types):
        return config.format(**environ)

    for key, value in iteritems(config):
        if not isinstance(value, string_types):
            continue
        config[key] = value.format(**environ)

    return config_to_args(config)

def print_error(message, wrap=True):
    """
    Print error message to stderr, using ANSI-colors.
    """
    if wrap:
        message = 'ERROR: {0}. Exit...'.format(message.rstrip('.'))

    colorizer = (_color_wrap(colorama.Fore.RED)
                 if colorama
                 else lambda message: message)
    return print(colorizer(message), file=sys.stderr)

def read_config(filename, args):
    """
    Read and parse configuration file. By default, ``filename`` is relative
    path to current work directory.

    If no config file found, default ``CONFIG`` would be used.
    """
    # Initial vars
    config = defaultdict(dict)
    splitter = operator.methodcaller('split', ' ')

    converters = {
        __script__: {
            'env': safe_path,
            'pre_requirements': splitter,
        },
        'pip': {
            'allow_external': splitter,
            'allow_unverified': splitter,
        }
    }
    default = copy.deepcopy(CONFIG)
    sections = set(iterkeys(default))

    # Expand user and environ vars in config filename
    filename = os.path.expandvars(os.path.expanduser(filename))

    # Read config if it exists on disk
    if os.path.isfile(filename):
        parser = SafeConfigParser()

        try:
            parser.read(filename)
        except ConfigParserError:
            print_error('Cannot parse config file at {0!r}'.format(filename))
            return None

        # Apply config for each possible section
        for section in sections:
            if not parser.has_section(section):
                continue

            items = parser.items(section)

            # Make auto convert here for integers and boolean values
            for key, value in items:
                try:
                    value = int(value)
                except (TypeError, ValueError):
                    try:
                        value = bool(strtobool(value))
                    except ValueError:
                        pass

                if section in converters and key in converters[section]:
                    value = converters[section][key](value)

                config[section][key] = value

    # Update config with default values if necessary
    for section, data in iteritems(default):
        if section not in config:
            config[section] = data
        else:
            for key, value in iteritems(data):
                config[section].setdefault(key, value)

    # Update bootstrap config from parsed args
    keys = set((
        'env', 'hook', 'ignore_activated', 'pre_requirements', 'quiet',
        'recreate', 'requirements'
    ))

    for key in keys:
        value = getattr(args, key)
        config[__script__].setdefault(key, value)

        if key == 'pre_requirements' and not value:
            continue

        if value is not None:
            config[__script__][key] = value

    return config

def run_cmd(cmd, echo=False, fail_silently=False, **kwargs):
    """
    Call given tuple or string with ``subprocess.call`` function.

    If ``echo`` show command to call and its output in STDOUT, otherwise hide
    all output.
    """
    if echo:
        cmd_str = cmd if isinstance(cmd, string_types) else ' '.join(cmd)
        kwargs['stdout'], kwargs['stderr'] = sys.stdout, sys.stderr
        print('$ {0}'.format(cmd_str))
    else:
        out, err = get_temp_streams()
        kwargs['stdout'], kwargs['stderr'] = out, err

    try:
        retcode = subprocess.call(cmd, **kwargs)
    except subprocess.CalledProcessError as err:
        if fail_silently:
            return False
        print_error(str(err) if IS_PY3 else unicode(err))  # noqa

    if retcode and echo and not fail_silently:
        print_error('Command {0!r} returned non-zero exit status {1}'.
                    format(cmd_str, retcode))

    return retcode

def run_hook(hook, config, quiet=False):
    """
    Run post-bootstrap hook if any.
    """
    if not hook:
        return True

    if not quiet:
        print('== Step 3. Run post-bootstrap hook ==')

    result = not run_cmd(prepare_args(hook, config),
                         echo=not quiet,
                         fail_silently=True,
                         shell=True)

    if not quiet:
        print()

    return result

def save_traceback(err):
    """
    Save error traceback to bootstrapper log file.
    """
    # Store logs to ~/.bootstrapper directory
    dirname = safe_path(os.path.expanduser(
        os.path.join('~', '.{0}'.format(__script__))
    ))

    # But ensure that directory exists
    if not os.path.isdir(dirname):
        os.mkdir(dirname)

    # Now we ready to put traceback to log file
    filename = os.path.join(dirname, '{0}.log'.format(__script__))

    with open(filename, 'a+') as handler:
        traceback.print_exc(file=handler)

    # And show colorized message
    message = ('User aborted workflow'
               if isinstance(err, KeyboardInterrupt)
               else 'Unexpected error catched')
    print_error(message)
    print_error('Full log stored to {0}'.format(filename), False)

    return True

def smart_str(value, encoding='utf-8', errors='strict'):
    """
    Convert Python object to string.
    """
    if not IS_PY3 and isinstance(value, unicode):  # noqa
        return value.encode(encoding, errors)
    return str(value)

def which(executable):
    """
    Shortcut to check whether executable available in current environment or
    not.
    """
    cmd = 'where' if IS_WINDOWS else 'which'
    return not run_cmd((cmd, executable), fail_silently=True)

if __name__ == '__main__':
    sys.exit(int(main()))
<|endoftext|>