<commit_before>import re
import sys
import time
import errno
import multiprocessing
import logging

from xopen import xopen
import dnaio

try:
    import resource
except ImportError:
    # Windows
    resource = None

logger = logging.getLogger(__name__)

def available_cpu_count():
    """
    Return the number of available virtual or physical CPUs on this system.
    The number of available CPUs can be smaller than the total number of CPUs
    when the cpuset(7) mechanism is in use, as is the case on some cluster
    systems.

    Adapted from http://stackoverflow.com/a/1006301/715090
    """
    try:
        with open("/proc/self/status") as f:
            status = f.read()
        m = re.search(r"(?m)^Cpus_allowed:\s*(.*)$", status)
        if m:
            res = bin(int(m.group(1).replace(",", ""), 16)).count("1")
            if res > 0:
                return min(res, multiprocessing.cpu_count())
    except IOError:
        pass

    return multiprocessing.cpu_count()

def raise_open_files_limit(n):
    if resource is None:
        return
    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    soft = min(soft + n, hard)
    resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))

class Progress:
    """
    Print an animated progress report to sys.stderr
    """

    def __init__(self, every=1):
        """
        every: minimum time to wait in seconds between progress updates
        """
        self._every = every
        self._n = 0
        # Time at which the progress was last updated
        self._time = time.time()
        self._start_time = self._time
        self._animation = self.scissors()

    @staticmethod
    def scissors(width=10):
        while True:
            for is_reverse, rang in [(False, range(width + 1)), (True, range(width + 1))]:
                for position in rang:
                    for is_open in (True, False):
                        left = " " * position
                        right = "-" * (width - position)
                        if is_reverse:
                            sc = ">8" if is_open else "=8"
                            left, right = right, left
                        else:
                            sc = "8<" if is_open else "8="
                        yield "[" + left + sc + right + "]"

    def update(self, total, _final=False):
        current_time = time.time()
        if _final:
            time_delta = current_time - self._start_time
            delta = total
        else:
            time_delta = current_time - self._time
            delta = total - self._n
        if delta < 1:
            return
        if not _final:
            if time_delta < self._every:
                return
            if total <= self._n:
                return

        t = current_time - self._start_time
        hours = int(t) // 3600
        minutes = (int(t) - hours * 3600) // 60
        seconds = int(t) % 60
        per_second = delta / time_delta
        per_item = time_delta / delta

        print(
            "\r"
            "{animation} {hours:02d}:{minutes:02d}:{seconds:02d} "
            "{total:13,d} reads  @  {per_item:7.1F} µs/read; {per_minute:6.2F} M reads/minute"
            "".format(
                hours=hours, minutes=minutes, seconds=seconds,
                total=total, per_item=per_item * 1E6, per_minute=per_second * 60 / 1E6,
                animation=next(self._animation)),
            end="", file=sys.stderr
        )
        self._n = total
        self._time = current_time

    def stop(self, total):
        """
        Print final progress reflecting the final total
        """
        self.update(total, _final=True)
        print(file=sys.stderr)

class DummyProgress:
    """
    Has the same interface as Progress, but does not print anything
    """
    def update(self, total, _final=False):
        pass

    def stop(self, total):
        pass

_REVCOMP_TRANS = str.maketrans(
    "ACGTUMRWSYKVHDBNacgtumrwsykvhdbn",
    "TGCAAKYWSRMBDHVNtgcaakywsrmbdhvn",
)

def reverse_complement(s: str):
    return s.translate(_REVCOMP_TRANS)[::-1]

def reverse_complemented_sequence(sequence: dnaio.Sequence):
    if sequence.qualities is None:
        qualities = None
    else:
        qualities = sequence.qualities[::-1]
    return dnaio.Sequence(sequence.name, reverse_complement(sequence.sequence), qualities)

class FileOpener:
    def __init__(self, compression_level: int = 6, threads: int = None):
        self.compression_level = compression_level
        self.threads = threads

    def xopen(self, path, mode):
        logger.debug("Opening file '%s', mode '%s' with xopen", path, mode)
        return xopen(path, mode, compresslevel=self.compression_level, threads=self.threads)

    def xopen_or_none(self, path, mode):
        """Return opened file or None if the path is None"""
        if path is None:
            return None
        return self.xopen(path, mode)

    def xopen_pair(self, path1, path2, mode):
        if path1 is None and path2 is not None:
            raise ValueError("When giving paths for paired-end files, only providing the second"
                " file is not supported")
        file1 = self.xopen_or_none(path1, mode)
        file2 = self.xopen_or_none(path2, mode)
        return file1, file2

    def dnaio_open(self, *args, **kwargs):
        logger.debug("Opening file '%s', mode '%s' with dnaio", args[0], kwargs['mode'])
        kwargs["opener"] = self.xopen
        return dnaio.open(*args, **kwargs)

    def dnaio_open_raise_limit(self, path, qualities):
        """
        Open a FASTA/FASTQ file for writing. If it fails because the number of open files
        would be exceeded, try to raise the soft limit and re-try.
        """
        try:
            f = self.dnaio_open(path, mode="w", qualities=qualities)
        except OSError as e:
            if e.errno == errno.EMFILE:  # Too many open files
                raise_open_files_limit(8)
                f = self.dnaio_open(path, mode="w", qualities=qualities)
            else:
                raise
        return f
<commit_msg>Fix type checking problem<commit_after>import re
import sys
import time
import errno
import multiprocessing
import logging

from xopen import xopen
import dnaio

try:
    import resource
except ImportError:
    # Windows
    resource = None  # type: ignore

logger = logging.getLogger(__name__)

def available_cpu_count():
    """
    Return the number of available virtual or physical CPUs on this system.
    The number of available CPUs can be smaller than the total number of CPUs
    when the cpuset(7) mechanism is in use, as is the case on some cluster
    systems.

    Adapted from http://stackoverflow.com/a/1006301/715090
    """
    try:
        with open("/proc/self/status") as f:
            status = f.read()
        m = re.search(r"(?m)^Cpus_allowed:\s*(.*)$", status)
        if m:
            res = bin(int(m.group(1).replace(",", ""), 16)).count("1")
            if res > 0:
                return min(res, multiprocessing.cpu_count())
    except IOError:
        pass

    return multiprocessing.cpu_count()

def raise_open_files_limit(n):
    if resource is None:
        return
    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    soft = min(soft + n, hard)
    resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))

class Progress:
    """
    Print an animated progress report to sys.stderr
    """

    def __init__(self, every=1):
        """
        every: minimum time to wait in seconds between progress updates
        """
        self._every = every
        self._n = 0
        # Time at which the progress was last updated
        self._time = time.time()
        self._start_time = self._time
        self._animation = self.scissors()

    @staticmethod
    def scissors(width=10):
        while True:
            for is_reverse, rang in [(False, range(width + 1)), (True, range(width + 1))]:
                for position in rang:
                    for is_open in (True, False):
                        left = " " * position
                        right = "-" * (width - position)
                        if is_reverse:
                            sc = ">8" if is_open else "=8"
                            left, right = right, left
                        else:
                            sc = "8<" if is_open else "8="
                        yield "[" + left + sc + right + "]"

    def update(self, total, _final=False):
        current_time = time.time()
        if _final:
            time_delta = current_time - self._start_time
            delta = total
        else:
            time_delta = current_time - self._time
            delta = total - self._n
        if delta < 1:
            return
        if not _final:
            if time_delta < self._every:
                return
            if total <= self._n:
                return

        t = current_time - self._start_time
        hours = int(t) // 3600
        minutes = (int(t) - hours * 3600) // 60
        seconds = int(t) % 60
        per_second = delta / time_delta
        per_item = time_delta / delta

        print(
            "\r"
            "{animation} {hours:02d}:{minutes:02d}:{seconds:02d} "
            "{total:13,d} reads  @  {per_item:7.1F} µs/read; {per_minute:6.2F} M reads/minute"
            "".format(
                hours=hours, minutes=minutes, seconds=seconds,
                total=total, per_item=per_item * 1E6, per_minute=per_second * 60 / 1E6,
                animation=next(self._animation)),
            end="", file=sys.stderr
        )
        self._n = total
        self._time = current_time

    def stop(self, total):
        """
        Print final progress reflecting the final total
        """
        self.update(total, _final=True)
        print(file=sys.stderr)

class DummyProgress:
    """
    Has the same interface as Progress, but does not print anything
    """
    def update(self, total, _final=False):
        pass

    def stop(self, total):
        pass

_REVCOMP_TRANS = str.maketrans(
    "ACGTUMRWSYKVHDBNacgtumrwsykvhdbn",
    "TGCAAKYWSRMBDHVNtgcaakywsrmbdhvn",
)

def reverse_complement(s: str):
    return s.translate(_REVCOMP_TRANS)[::-1]

def reverse_complemented_sequence(sequence: dnaio.Sequence):
    if sequence.qualities is None:
        qualities = None
    else:
        qualities = sequence.qualities[::-1]
    return dnaio.Sequence(sequence.name, reverse_complement(sequence.sequence), qualities)

class FileOpener:
    def __init__(self, compression_level: int = 6, threads: int = None):
        self.compression_level = compression_level
        self.threads = threads

    def xopen(self, path, mode):
        logger.debug("Opening file '%s', mode '%s' with xopen", path, mode)
        return xopen(path, mode, compresslevel=self.compression_level, threads=self.threads)

    def xopen_or_none(self, path, mode):
        """Return opened file or None if the path is None"""
        if path is None:
            return None
        return self.xopen(path, mode)

    def xopen_pair(self, path1, path2, mode):
        if path1 is None and path2 is not None:
            raise ValueError("When giving paths for paired-end files, only providing the second"
                " file is not supported")
        file1 = self.xopen_or_none(path1, mode)
        file2 = self.xopen_or_none(path2, mode)
        return file1, file2

    def dnaio_open(self, *args, **kwargs):
        logger.debug("Opening file '%s', mode '%s' with dnaio", args[0], kwargs['mode'])
        kwargs["opener"] = self.xopen
        return dnaio.open(*args, **kwargs)

    def dnaio_open_raise_limit(self, path, qualities):
        """
        Open a FASTA/FASTQ file for writing. If it fails because the number of open files
        would be exceeded, try to raise the soft limit and re-try.
        """
        try:
            f = self.dnaio_open(path, mode="w", qualities=qualities)
        except OSError as e:
            if e.errno == errno.EMFILE:  # Too many open files
                raise_open_files_limit(8)
                f = self.dnaio_open(path, mode="w", qualities=qualities)
            else:
                raise
        return f
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
import transforms
from contextlib import contextmanager

recompile_keys = ['prior', 'transform', 'fixed'] # when one of these attributes is set, notify a recompilation

class Parentable(object):
    """
    A very simple class for objects in a tree, where each node contains a
    reference to '_parent'.

    This class can figure out its own name (by seeing what it's called by the
    _parent's __dict__) and also recurse up to the highest_parent.
    """
    _parent = None
    @property
    def highest_parent(self):
        if self._parent is None:
            return self
        else:
            return self._parent.highest_parent
    @property
    def name(self):
        """to get the name of this object, have a look at what our _parent has called us"""
        if self._parent is None:
            return 'unnamed'
        if isinstance(self._parent, ParamList):
            return 'item%i'%self._parent._list.index(self)
        matches = [key for key, value in self._parent.__dict__.items() if value is self]
        if len(matches) == 0:
            raise ValueError, "mis-specified parent. This param's _parent does not contain a reference to it."
        if len(matches) > 1:
            raise ValueError, "This param appears to be doubly referenced by a parent"
        return matches [0]

class Param(Parentable):
    """
    An object to represent parameters.

    Getting and setting values
    --
    The current value of the parameter is stored in self._array as a
    numpy.ndarray.  Changing the value of the Param is as simple as assignment
    (once the Param is part of a model). Example:

    >>> m = GPflow.model.Model()
    >>> m.p = GPflow.param.Param(1.0)
    >>> print(m)
    model.p transform:(none) prior:None
    [ 1.]
    >>> m.p = 3.2
    >>> print(m)
    model.p transform:(none) prior:None
    [ 3.2]

    Unconstrained optimization
    --
    The parameter can be transformed to a 'free state' where it
    can be optimized. The methods

    >>> self.get_free_state
    >>> self.set_state

    transforms between self._array and the free state.

    To apply a transform to the Param, simply set the transform atribute with a GPflow.transforms object
    >>> m = GPflow.model.Model()
    >>> m.p = GPflow.param.Param(1.0)
    >>> print(m)
    model.p transform:(none) prior:None
    [ 1.]
    >>> m.p.transform = GPflow.transforms.Exp()
    >>> print(m)
    model.p transform:+ve prior:None
    [ 1.]

    Fixes
    --
    There is a self._fixed flag, in which case the parameter does not get
    optimized. To enable this, during make_tf_array, the fixed values of
    the parameter are returned. Fixes and transforms can be used together, in
    the sense that fixes tkae priority over transforms, so unfixing a parameter
    is as simple as setting the flag. Example:

    >>> p = Param(1.0, transform=GPflow.transforms.positive)
    >>> m = GPflow.model.Model()
    >>> m.p = p # the model has a sinlge parameter, constrained to be +ve
    >>> m.p.fixed = True # the model now has no free parameters
    >>> m.p.fixed = False # the model has a sinlge parameter, constrained to be +ve

    Compiling into tensorflow
    --
    The method

    >>> self.make_tf_array

    constructs a tensorflow representation of the parameter, from a tensorflow vector representing the free state.

    The `self.prior` object is used to place priors on prameters, and the
    `self.transform` object is used to enable unconstrained optimization and
    mcmc.

    """
    def __init__(self, array, transform=transforms.Identity()):
        Parentable.__init__(self)
        self._array = np.asarray(np.atleast_1d(array), dtype=np.float64)
        self.transform = transform
        self.prior = None
        self.fixed = False

    def make_tf_array(self, free_array):
        """
        free_array is a tensorflow vector which will be the optimisation target,
        i.e. it will be free to take any value.

        Here we take that array, and transform and reshape it so that it can be
        used to represent this parameter

        Then we return the number of elements that we've used to construct the
        array, so that it can be sliced fo rthe next Param.
        """

        #TODO what about constraints that change the size ??

        if self.fixed:
            self._tf_array = self._array.copy()
            return 0
        x_free = free_array[:self.size]
        mapped_array = self.transform.tf_forward(x_free)
        self._tf_array = tf.reshape(mapped_array, self.shape)
        self._log_jacobian = self.transform.tf_log_jacobian(x_free)
        return self.size

    def get_free_state(self):
        """
        Take the current state of this variable, as stored in self._array, and transform it to the 'free' state.

        This is a numpy method.
        """
        if self.fixed: return np.empty((0,))
        return self.transform.backward(self._array.flatten())

    def set_state(self, x):
        """
        Given a vector x representing the 'free' state of this param, transform
        it 'forwards' and store the result in self._array.

        This is a numpy method.
        """
        if self.fixed: return 0
        new_array = self.transform.forward(x[:self.size]).reshape(self.shape)
        assert new_array.shape == self.shape
        self._array[...] = new_array
        return self.size

    def build_prior(self):
        """
        Build a tensorflow representation of the prior density. The log jacobian is included.
        """
        if self.prior is None:
            return 0
        elif self._tf_array is None:
            raise ValueError, "tensorflow array has not been initialized"
        else:
            return self.prior.logp(self._tf_array) + self._log_jacobian

    def __setattr__(self, key, value):
        """
        When some attirbutes are set, we need to recompile the tf model before evaluation.
        """
        object.__setattr__(self, key, value)
        if key in recompile_keys:
            self.highest_parent._needs_recompile = True

    def __str__(self, prepend=''):
        return prepend + \
                '\033[1m' + self.name + '\033[0m' + \
                ' transform:' + str(self.transform) + \
                ' prior:' + str(self.prior) + \
                (' [FIXED]' if self.fixed else '') + \
                '\n' + str(self._array)

    @property
    def size(self):
        return self._array.size

    @property
    def shape(self):
        return self._array.shape

    def _html_table_rows(self, name_prefix=''):
        """
        Construct a row of an html table, to be used in the jupyter notebook.
        """
        html = "
"
        html += "{0}	".format(name_prefix + self.name)
        html += "{0}	".format(str(self._array).replace('\n', '
'))
        html += "{0}	".format(str(self.prior))
        html += "{0}	".format('[FIXED]' if self.fixed else str(self.transform))
        html += "
"
        return html

class Parameterized(Parentable):
    """
    An object to contain parameters.

    This object is designed to be part of a tree, with Param objects at the
    leaves. We can then recurse down the tree to find all the parameters
    (leaves), or recurse up the tree (using highest_parent) from the leaves to
    the root.

    A useful application of such a recursion is 'tf_mode', where the
    parameters appear as their _tf_array variables. This allows us to build
    models on those parameters. During _tf_mode, the __getattribute__
    method is overwritten to return tf arrays in place of parameters.

    Another recurseive function is build_prior wich sums the log-prior from all
    of the tree's parameters (whilst in tf_mode!).
    """
    def __init__(self):
        Parentable.__init__(self)
        self._tf_mode = False

    def __getattribute__(self, key):
        """
        Here, we overwrite the getattribute method.

        If tf mode is off, this does nothing.

        If tf mode is on, all child parameters will appear as their tf
        representations.
        """
        o =  object.__getattribute__(self, key)
        if isinstance(o, Param) and object.__getattribute__(self, '_tf_mode'):
            return o._tf_array
        return o

    def __setattr__(self, key, value):
        """
        When a value is assigned to a Param, put that value in the
        Param's array (rather than just overwriting that Param with the
        new value). i.e. this

        >>> p = Parameterized()
        >>> p.p = Param(1.0)
        >>> p.p = 2.0

        should be equivalent to this

        >>> p = Parameterized()
        >>> p.p = Param(1.0)
        >>> p.p._array[...] = 2.0

        Additionally, when Param or Parameterized objects are added, let them
        know that this node is the _parent
        """

        #set the _array value of child nodes instead of standard assignment.
        if key in self.__dict__.keys():
            p = getattr(self, key)
            if isinstance(p, Param):
                p._array[...] = value
                return # don't call object.setattr or set the _parent value

        #use the standard setattr
        object.__setattr__(self, key, value)

        #make sure a new child node knows this is the _parent:
        if isinstance(value, (Param, Parameterized)) and (key is not '_parent'):
            value._parent = self

    def make_tf_array(self, X):
        """
        X is a tf. placeholder. It gets passed to all the children of
        this class (that are Parameterized or Param objects), which then
        construct their tf_array variables from consecutive sections.
        """
        count = 0
        for p in self.sorted_params:
            count += p.make_tf_array(X[count:])
        return count

    @property
    def sorted_params(self):
        """
        Return a list of all the child parameters, sorted by id. This makes
        sure they're always in the same order.
        """
        params = [child for key, child in self.__dict__.items() if isinstance(child, (Param, Parameterized)) and key is not '_parent']
        return sorted(params, key=id)

    def get_free_state(self):
        """
        recurse get_free_state on all child parameters, and hstack them.
        """
        return np.hstack([p.get_free_state() for p in self.sorted_params] + [np.empty(0)]) # additional empty array allows hstacking of empty list

    def set_state(self, x):
        """
        Set the values of all the parameters by recursion
        """
        count = 0
        for p in self.sorted_params:
            count += p.set_state(x[count:])
        return count

    @contextmanager
    def tf_mode(self):
        """
        A context for building models. Correct usage is

        with m.tf_mode:
            #do tf stuff, lik
            m.build_likelihood()
            m.build_prior()

        with this context engaged, any Param objects which are children of this
        class will appear as their tf-variables. Example

        >>> m = Parameterized()
        >>> m.foo = Param(1.0)
        >>> m.make_tf_array(tt.dvector())
        >>> print m.foo
        foo
        [ 1.]
        >>> with m.tf_mode():
        >>>     print m.foo
        Reshape{1}.0

        The idea is that in tf_mode, we can easily get refrences to the
        tf representation of parameters in order to construct tf
        objective functions.
        """
        self._begin_tf_mode()
        yield
        self._end_tf_mode()

    def _begin_tf_mode(self):
        [child._begin_tf_mode() for child in self.sorted_params if isinstance(child, Parameterized)]
        self._tf_mode = True

    def _end_tf_mode(self):
        [child._end_tf_mode() for child in self.sorted_params if isinstance(child, Parameterized)]
        self._tf_mode = False

    def build_prior(self):
        """
        Build a tf expression for the prior by summing all child-node priors.
        """
        return sum([p.build_prior() for p in self.sorted_params])

    def __str__(self, prepend=''):
        prepend += self.name + '.'
        return '\n'.join([p.__str__(prepend) for p in self.sorted_params])

    def _html_table_rows(self, name_prefix=''): # pragma: no cover
        """
        Get the rows of the html table for this object
        """
        name_prefix +=  self.name + '.'
        return ''.join([p._html_table_rows(name_prefix) for p in self.sorted_params])

    def _repr_html_(self):
        """
        Build a small html table for display in the jupyter notebook.
        """
        html = ["
"]

        #build the header
        header = "
"
        header += "Name	"
        header += "values	"
        header += "prior	"
        header += "constriant	"
        header += "
"
        html.append(header)

        html.append(self._html_table_rows())

        html.append("
")
        return ''.join(html)

class ParamList(Parameterized):
    """
    A list of parameters. This allows us to store parameters in a list whilst
    making them 'visible' to the GPflow machinery.

    The correct usage is

    >>> my_list = GPflow.param.ParamList([Param1, Param2])

    You can then iterate through the list. For example, to compute the sum:
    >>> my_sum = reduce(tf.add, my_list)

    or the sum of the squares:
    >>> rmse = tf.sqrt(reduce(tf.add, map(tf.square, my_list)))

    You can append things:
    >>> my_list.append(GPflow.kernels.RBF(1))

    but only if the are Parameters (or Parameterized objects). You can set the
    value of Parameters in the list:

    >>> my_list = GPflow.param.ParamList([GPflow.param.Param(2)])
    >>> print my_list
    unnamed.item0 transform:(none) prior:None
    [ 2.]
    >>> my_list[0] = 12
    >>> print my_list
    unnamed.item0 transform:(none) prior:None
    [ 12.]

    But you can't change elements of the list by assignment:
    >>> my_list = GPflow.param.ParamList([GPflow.param.Param(2)])
    >>> new_param = GPflow.param.Param(4)
    >>> my_list[0] = new_param # raises exception

    """
    def __init__(self, list_of_params=[]):
        Parameterized.__init__(self)
        for item in list_of_params:
            assert isinstance(item, (Param, Parameterized))
            item._parent = self
        self._list = list_of_params

    @property
    def sorted_params(self):
        return self._list

    def __getitem__(self, key):
        """
        If tf mode is off, this simply returns the corresponding Param .

        If tf mode is on, all items will appear as their tf
        representations.
        """
        o = self.sorted_params[key]
        if isinstance(o, Param) and self._tf_mode:
            return o._tf_array
        return o

    def append(self, item):
        assert isinstance(item, (Param, Parameterized)), "this object is for containing parameters"
        item._parent = self
        self.sorted_params.append(item)

    def __setitem__(self, key, value):
        """
        It's not possible to assign to things in the list, but it is possbile
        to set their values by assignment.
        """
        self.sorted_params[key]._array[...] = value
<commit_msg>param.py is now pep8 compliant<commit_after>import numpy as np
import tensorflow as tf
import transforms
from contextlib import contextmanager

# when one of these attributes is set, notify a recompilation
recompile_keys = ['prior', 'transform', 'fixed']

class Parentable(object):
    """
    A very simple class for objects in a tree, where each node contains a
    reference to '_parent'.

    This class can figure out its own name (by seeing what it's called by the
    _parent's __dict__) and also recurse up to the highest_parent.
    """
    _parent = None

    @property
    def highest_parent(self):
        if self._parent is None:
            return self
        else:
            return self._parent.highest_parent

    @property
    def name(self):
        """to get the name of this object, have a look at
        what our _parent has called us"""
        if self._parent is None:
            return 'unnamed'
        if isinstance(self._parent, ParamList):
            return 'item%i' % self._parent._list.index(self)
        matches = [key for key, value in self._parent.__dict__.items()
                   if value is self]
        if len(matches) == 0:
            raise ValueError("mis-specified parent. This param's _parent does\
                               not contain a reference to it.")
        if len(matches) > 1:
            raise ValueError("This param appears to be doubly\
                              referenced by a parent")
        return matches[0]

class Param(Parentable):
    """
    An object to represent parameters.

    Getting and setting values
    --
    The current value of the parameter is stored in self._array as a
    numpy.ndarray.  Changing the value of the Param is as simple as assignment
    (once the Param is part of a model). Example:

    >>> m = GPflow.model.Model()
    >>> m.p = GPflow.param.Param(1.0)
    >>> print(m)
    model.p transform:(none) prior:None
    [ 1.]
    >>> m.p = 3.2
    >>> print(m)
    model.p transform:(none) prior:None
    [ 3.2]

    Unconstrained optimization
    --
    The parameter can be transformed to a 'free state' where it
    can be optimized. The methods

    >>> self.get_free_state
    >>> self.set_state

    transforms between self._array and the free state.

    To apply a transform to the Param, simply set the transform atribute
    with a GPflow.transforms object
    >>> m = GPflow.model.Model()
    >>> m.p = GPflow.param.Param(1.0)
    >>> print(m)
    model.p transform:(none) prior:None
    [ 1.]
    >>> m.p.transform = GPflow.transforms.Exp()
    >>> print(m)
    model.p transform:+ve prior:None
    [ 1.]

    Fixes
    --
    There is a self._fixed flag, in which case the parameter does not get
    optimized. To enable this, during make_tf_array, the fixed values of
    the parameter are returned. Fixes and transforms can be used together, in
    the sense that fixes tkae priority over transforms, so unfixing a parameter
    is as simple as setting the flag. Example:

    >>> p = Param(1.0, transform=GPflow.transforms.positive)
    >>> m = GPflow.model.Model()
    >>> m.p = p # the model has a sinlge parameter, constrained to be +ve
    >>> m.p.fixed = True # the model now has no free parameters
    >>> m.p.fixed = False # the model has a sinlge parameter, constrained +ve

    Compiling into tensorflow
    --
    The method

    >>> self.make_tf_array

    constructs a tensorflow representation of the parameter, from a tensorflow
    vector representing the free state.

    The `self.prior` object is used to place priors on prameters, and the
    `self.transform` object is used to enable unconstrained optimization and
    mcmc.
    """
    def __init__(self, array, transform=transforms.Identity()):
        Parentable.__init__(self)
        self._array = np.asarray(np.atleast_1d(array), dtype=np.float64)
        self.transform = transform
        self.prior = None
        self.fixed = False

    def make_tf_array(self, free_array):
        """
        free_array is a tensorflow vector which will be the optimisation
        target, i.e. it will be free to take any value.

        Here we take that array, and transform and reshape it so that it can be
        used to represent this parameter

        Then we return the number of elements that we've used to construct the
        array, so that it can be sliced fo rthe next Param.
        """

        # TODO what about constraints that change the size ??

        if self.fixed:
            self._tf_array = self._array.copy()
            return 0
        x_free = free_array[:self.size]
        mapped_array = self.transform.tf_forward(x_free)
        self._tf_array = tf.reshape(mapped_array, self.shape)
        self._log_jacobian = self.transform.tf_log_jacobian(x_free)
        return self.size

    def get_free_state(self):
        """
        Take the current state of this variable, as stored in self._array, and
        transform it to the 'free' state.

        This is a numpy method.
        """
        if self.fixed:
            return np.empty((0,))
        return self.transform.backward(self._array.flatten())

    def set_state(self, x):
        """
        Given a vector x representing the 'free' state of this param, transform
        it 'forwards' and store the result in self._array.

        This is a numpy method.
        """
        if self.fixed:
            return 0
        new_array = self.transform.forward(x[:self.size]).reshape(self.shape)
        assert new_array.shape == self.shape
        self._array[...] = new_array
        return self.size

    def build_prior(self):
        """
        Build a tensorflow representation of the prior density.
        The log jacobian is included.
        """
        if self.prior is None:
            return 0
        elif self._tf_array is None:
            raise ValueError("tensorflow array has not been initialized")
        else:
            return self.prior.logp(self._tf_array) + self._log_jacobian

    def __setattr__(self, key, value):
        """
        When some attirbutes are set, we need to recompile the tf model before
        evaluation.
        """
        object.__setattr__(self, key, value)
        if key in recompile_keys:
            self.highest_parent._needs_recompile = True

    def __str__(self, prepend=''):
        return prepend + \
            '\033[1m' + self.name + '\033[0m' + \
            ' transform:' + str(self.transform) + \
            ' prior:' + str(self.prior) + \
            (' [FIXED]' if self.fixed else '') + \
            '\n' + str(self._array)

    @property
    def size(self):
        return self._array.size

    @property
    def shape(self):
        return self._array.shape

    def _html_table_rows(self, name_prefix=''):
        """
        Construct a row of an html table, to be used in the jupyter notebook.
        """
        html = "
"
        html += "{0}	".format(name_prefix + self.name)
        html += "{0}	".format(str(self._array).replace('\n', '
'))
        html += "{0}	".format(str(self.prior))
        html += "{0}	".format('[FIXED]' if self.fixed
                                      else str(self.transform))
        html += "
"
        return html

class Parameterized(Parentable):
    """
    An object to contain parameters.

    This object is designed to be part of a tree, with Param objects at the
    leaves. We can then recurse down the tree to find all the parameters
    (leaves), or recurse up the tree (using highest_parent) from the leaves to
    the root.

    A useful application of such a recursion is 'tf_mode', where the
    parameters appear as their _tf_array variables. This allows us to build
    models on those parameters. During _tf_mode, the __getattribute__
    method is overwritten to return tf arrays in place of parameters.

    Another recurseive function is build_prior wich sums the log-prior from all
    of the tree's parameters (whilst in tf_mode!).
    """
    def __init__(self):
        Parentable.__init__(self)
        self._tf_mode = False

    def __getattribute__(self, key):
        """
        Here, we overwrite the getattribute method.

        If tf mode is off, this does nothing.

        If tf mode is on, all child parameters will appear as their tf
        representations.
        """
        o = object.__getattribute__(self, key)
        if isinstance(o, Param) and object.__getattribute__(self, '_tf_mode'):
            return o._tf_array
        return o

    def __setattr__(self, key, value):
        """
        When a value is assigned to a Param, put that value in the
        Param's array (rather than just overwriting that Param with the
        new value). i.e. this

        >>> p = Parameterized()
        >>> p.p = Param(1.0)
        >>> p.p = 2.0

        should be equivalent to this

        >>> p = Parameterized()
        >>> p.p = Param(1.0)
        >>> p.p._array[...] = 2.0

        Additionally, when Param or Parameterized objects are added, let them
        know that this node is the _parent
        """

        # set the _array value of child nodes instead of standard assignment.
        if key in self.__dict__.keys():
            p = getattr(self, key)
            if isinstance(p, Param):
                p._array[...] = value
                return  # don't call object.setattr or set the _parent value

        # use the standard setattr
        object.__setattr__(self, key, value)

        # make sure a new child node knows this is the _parent:
        if isinstance(value, (Param, Parameterized)) and key is not '_parent':
            value._parent = self

    def make_tf_array(self, X):
        """
        X is a tf. placeholder. It gets passed to all the children of
        this class (that are Parameterized or Param objects), which then
        construct their tf_array variables from consecutive sections.
        """
        count = 0
        for p in self.sorted_params:
            count += p.make_tf_array(X[count:])
        return count

    @property
    def sorted_params(self):
        """
        Return a list of all the child parameters, sorted by id. This makes
        sure they're always in the same order.
        """
        params = [child for key, child in self.__dict__.items()
                  if isinstance(child, (Param, Parameterized))
                  and key is not '_parent']
        return sorted(params, key=id)

    def get_free_state(self):
        """
        recurse get_free_state on all child parameters, and hstack them.
        """
        # Here, additional empty array allows hstacking of empty list
        return np.hstack([p.get_free_state() for p in self.sorted_params]
                         + [np.empty(0)])

    def set_state(self, x):
        """
        Set the values of all the parameters by recursion
        """
        count = 0
        for p in self.sorted_params:
            count += p.set_state(x[count:])
        return count

    @contextmanager
    def tf_mode(self):
        """
        A context for building models. Correct usage is

        with m.tf_mode:
            #do tf stuff, lik
            m.build_likelihood()
            m.build_prior()

        with this context engaged, any Param objects which are children of this
        class will appear as their tf-variables. Example

        >>> m = Parameterized()
        >>> m.foo = Param(1.0)
        >>> m.make_tf_array(tt.dvector())
        >>> print m.foo
        foo
        [ 1.]
        >>> with m.tf_mode():
        >>>     print m.foo
        Reshape{1}.0

        The idea is that in tf_mode, we can easily get refrences to the
        tf representation of parameters in order to construct tf
        objective functions.
        """
        self._begin_tf_mode()
        yield
        self._end_tf_mode()

    def _begin_tf_mode(self):
        [child._begin_tf_mode() for child in self.sorted_params
         if isinstance(child, Parameterized)]
        self._tf_mode = True

    def _end_tf_mode(self):
        [child._end_tf_mode() for child in self.sorted_params
         if isinstance(child, Parameterized)]
        self._tf_mode = False

    def build_prior(self):
        """
        Build a tf expression for the prior by summing all child-node priors.
        """
        return sum([p.build_prior() for p in self.sorted_params])

    def __str__(self, prepend=''):
        prepend += self.name + '.'
        return '\n'.join([p.__str__(prepend) for p in self.sorted_params])

    def _html_table_rows(self, name_prefix=''):  # pragma: no cover
        """
        Get the rows of the html table for this object
        """
        name_prefix += self.name + '.'
        return ''.join([p._html_table_rows(name_prefix)
                        for p in self.sorted_params])

    def _repr_html_(self):
        """
        Build a small html table for display in the jupyter notebook.
        """
        html = ["
"]

        # build the header
        header = "
"
        header += "Name	"
        header += "values	"
        header += "prior	"
        header += "constriant	"
        header += "
"
        html.append(header)

        html.append(self._html_table_rows())

        html.append("
")
        return ''.join(html)

class ParamList(Parameterized):
    """
    A list of parameters. This allows us to store parameters in a list whilst
    making them 'visible' to the GPflow machinery.

    The correct usage is

    >>> my_list = GPflow.param.ParamList([Param1, Param2])

    You can then iterate through the list. For example, to compute the sum:
    >>> my_sum = reduce(tf.add, my_list)

    or the sum of the squares:
    >>> rmse = tf.sqrt(reduce(tf.add, map(tf.square, my_list)))

    You can append things:
    >>> my_list.append(GPflow.kernels.RBF(1))

    but only if the are Parameters (or Parameterized objects). You can set the
    value of Parameters in the list:

    >>> my_list = GPflow.param.ParamList([GPflow.param.Param(2)])
    >>> print my_list
    unnamed.item0 transform:(none) prior:None
    [ 2.]
    >>> my_list[0] = 12
    >>> print my_list
    unnamed.item0 transform:(none) prior:None
    [ 12.]

    But you can't change elements of the list by assignment:
    >>> my_list = GPflow.param.ParamList([GPflow.param.Param(2)])
    >>> new_param = GPflow.param.Param(4)
    >>> my_list[0] = new_param # raises exception

    """
    def __init__(self, list_of_params=[]):
        Parameterized.__init__(self)
        for item in list_of_params:
            assert isinstance(item, (Param, Parameterized))
            item._parent = self
        self._list = list_of_params

    @property
    def sorted_params(self):
        return self._list

    def __getitem__(self, key):
        """
        If tf mode is off, this simply returns the corresponding Param .

        If tf mode is on, all items will appear as their tf
        representations.
        """
        o = self.sorted_params[key]
        if isinstance(o, Param) and self._tf_mode:
            return o._tf_array
        return o

    def append(self, item):
        assert isinstance(item, (Param, Parameterized)),\
            "this object is for containing parameters"
        item._parent = self
        self.sorted_params.append(item)

    def __setitem__(self, key, value):
        """
        It's not possible to assign to things in the list, but it is possbile
        to set their values by assignment.
        """
        self.sorted_params[key]._array[...] = value
<|endoftext|>


<commit_before>from os import path
import sys
import json
import argparse

from docker.client import Client
from docker.utils import kwargs_from_env
from git import Repo, InvalidGitRepositoryError
import pkg_resources
import requests

class UserMessageException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

def _get_repo(directory):
    directory = path.abspath(directory)
    while directory is not '/':
        try:
            return Repo(directory)
        except InvalidGitRepositoryError:
            directory = path.dirname(directory)
    raise UserMessageException('To run this command you need to be in git source code directory.')

def _get_current_branch(repo):
    return repo.git.rev_parse('--abbrev-ref', 'HEAD')

def _check_branch(repo):
    if _get_current_branch(repo) != 'master':
        raise UserMessageException('Please switch to master branch to release the docker image.')

def _get_docker_image_name(docker_image_dir):
    docker_image_dir = path.abspath(docker_image_dir)
    docker_file_path = path.join(docker_image_dir, 'Dockerfile')
    if not path.exists(docker_file_path):
        raise UserMessageException('Unable to find Dockerfile at location %s' % docker_file_path)

    return (path.basename(path.dirname(docker_image_dir)), path.basename(docker_image_dir))

def _init_docker():
    kwargs = kwargs_from_env()

    if 'tls' in kwargs:
        # see http://docker-py.readthedocs.org/en/latest/boot2docker/
        import requests.packages.urllib3 as urllib3

        urllib3.disable_warnings()
        kwargs['tls'].assert_hostname = False

    docker = Client(**kwargs)
    try:
        docker.version()
    except:
        raise UserMessageException("Please set up 'docker' correctly")
    return docker

def _get_tags(organization, repository):
    response = requests.get('https://registry.hub.docker.com/v1/repositories/%s/%s/tags' % (organization, repository))
    print response
    if response.status_code == 404:
        return []
    if response.status_code != 200:
        raise UserMessageException('Unable to get tags for: %s/%s' % (organization, repository))

    tags = {}
    for entry in response.json():
        tags[entry['name']] = entry['layer']
    return tags

def _get_next_version(tags):
    version = 1
    while version in tags:
        version += 1

    return version

def _docker_build(docker, docker_image_dir, image):
    print "Building %s" % image
    for line in docker.build(docker_image_dir, image):
        print json.loads(line)['stream'][:-1]

def _docker_push(docker, image, tag):
    print "pushing %s:%s" % (image, tag)
    images = docker.images(image)
    if len(images) != 1:
        raise UserMessageException('Expected only one image for: %s' % image)
    docker.tag(images[0]['Id'], image[:image.find(':')], tag)
    for line in docker.push(image, tag, stream=True):
        line = json.loads(line)
        if 'error' in line:
            raise UserMessageException('While pushing: %s: %s' % (image, line['error']))
        elif 'id' in line:
            print '%s: %s' % (line['id'], line['status'])
        elif 'status' in line:
            print line['status']

def main():
    parser = argparse.ArgumentParser(description='Tool for releasing docker images.')
    parser.add_argument('--snapshot', '-s', action='store_true',
                        help="Release a snapshot version (push tags with git sha and updates 'latest')")
    parser.add_argument('--force', '-f', action='store_true',
                        help='Release even if it is already released')
    parser.add_argument('docker_image_dir', metavar='dir', nargs='+',
                        help='Location of docker image directory with Dockerfile to be released')
    parser.add_argument('--version', action='version', version=pkg_resources.require("docker-release")[0].version)
    args = parser.parse_args(sys.argv[1:])
    try:
        for docker_image_dir in args.docker_image_dir:
            organization, repository = _get_docker_image_name(docker_image_dir)
            repo = _get_repo(docker_image_dir)
            if not args.snapshot:
                _check_branch(repo)
            tags = _get_tags(organization, repository)
            hash_tag = repo.head.commit.hexsha[:7]
            if args.force == False and hash_tag in tags:
                raise UserMessageException('This version is already released')
            docker = _init_docker()
            image = '%s/%s:latest' % (organization, repository)
            _docker_build(docker, docker_image_dir, image)

            _docker_push(docker, image, hash_tag)
            if not args.snapshot:
                version = _get_next_version(tags)
                _docker_push(docker, image, version)
                repo.git.tag('%s/%s' % (image, version))
                repo.git.push('--tags')
            _docker_push(docker, image, 'latest')

    except UserMessageException, e:
        print "ERROR: %s" % e
        return 1
<commit_msg>Check if all is commited before building<commit_after>from os import path
import sys
import json
import argparse

from docker.client import Client
from docker.utils import kwargs_from_env
from git import Repo, InvalidGitRepositoryError
import pkg_resources
import requests

class UserMessageException(Exception):
    def __init__(self, value):
        self.value = value

def _get_repo(directory):
    directory = path.abspath(directory)
    while directory is not '/':
        try:
            repo = Repo(directory)
            if repo.is_dirty():
                raise UserMessageException('Please commit or stash all the changes')
            return repo
        except InvalidGitRepositoryError:
            directory = path.dirname(directory)
    raise UserMessageException('To run this command you need to be in git source code directory.')

def _get_current_branch(repo):
    return repo.git.rev_parse('--abbrev-ref', 'HEAD')

def _check_branch(repo):
    if _get_current_branch(repo) != 'master':
        raise UserMessageException('Please switch to master branch to release the docker image.')

def _get_docker_image_name(docker_image_dir):
    docker_image_dir = path.abspath(docker_image_dir)
    docker_file_path = path.join(docker_image_dir, 'Dockerfile')
    if not path.exists(docker_file_path):
        raise UserMessageException('Unable to find Dockerfile at location %s' % docker_file_path)

    return (path.basename(path.dirname(docker_image_dir)), path.basename(docker_image_dir))

def _init_docker():
    kwargs = kwargs_from_env()

    if 'tls' in kwargs:
        # see http://docker-py.readthedocs.org/en/latest/boot2docker/
        import requests.packages.urllib3 as urllib3

        urllib3.disable_warnings()
        kwargs['tls'].assert_hostname = False

    docker = Client(**kwargs)
    try:
        docker.version()
    except:
        raise UserMessageException("Please set up 'docker' correctly")
    return docker

def _get_tags(organization, repository):
    response = requests.get('https://registry.hub.docker.com/v1/repositories/%s/%s/tags' % (organization, repository))
    print response
    if response.status_code == 404:
        return []
    if response.status_code != 200:
        raise UserMessageException('Unable to get tags for: %s/%s' % (organization, repository))

    tags = {}
    for entry in response.json():
        tags[entry['name']] = entry['layer']
    return tags

def _get_next_version(tags):
    version = 1
    while version in tags:
        version += 1

    return version

def _docker_build(docker, docker_image_dir, image):
    print "Building %s" % image
    for line in docker.build(docker_image_dir, image):
        print json.loads(line)['stream'][:-1]

def _docker_push(docker, image, tag):
    print "pushing %s:%s" % (image, tag)
    images = docker.images(image)
    if len(images) != 1:
        raise UserMessageException('Expected only one image for: %s' % image)
    docker.tag(images[0]['Id'], image[:image.find(':')], tag)
    for line in docker.push(image, tag, stream=True):
        line = json.loads(line)
        if 'error' in line:
            raise UserMessageException('While pushing: %s: %s' % (image, line['error']))
        elif 'id' in line:
            print '%s: %s' % (line['id'], line['status'])
        elif 'status' in line:
            print line['status']

def main():
    parser = argparse.ArgumentParser(description='Tool for releasing docker images.')
    parser.add_argument('--snapshot', '-s', action='store_true',
                        help="Release a snapshot version (push tags with git sha and updates 'latest')")
    parser.add_argument('--force', '-f', action='store_true',
                        help='Release even if it is already released')
    parser.add_argument('docker_image_dir', metavar='dir', nargs='+',
                        help='Location of docker image directory with Dockerfile to be released')
    parser.add_argument('--version', action='version', version=pkg_resources.require("docker-release")[0].version)
    args = parser.parse_args(sys.argv[1:])
    try:
        for docker_image_dir in args.docker_image_dir:
            organization, repository = _get_docker_image_name(docker_image_dir)
            repo = _get_repo(docker_image_dir)
            if not args.snapshot:
                _check_branch(repo)
            tags = _get_tags(organization, repository)
            hash_tag = repo.head.commit.hexsha[:7]
            if args.force == False and hash_tag in tags:
                raise UserMessageException('This version is already released')
            docker = _init_docker()
            image = '%s/%s:latest' % (organization, repository)
            _docker_build(docker, docker_image_dir, image)

            _docker_push(docker, image, hash_tag)
            if not args.snapshot:
                version = _get_next_version(tags)
                _docker_push(docker, image, version)
                repo.git.tag('%s/%s' % (image, version))
                repo.git.push('--tags')
            _docker_push(docker, image, 'latest')

    except UserMessageException, e:
        print "ERROR: %s" % e.value
        return 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""An automatic command that handles subcommands in a git-like fashion."""

import inspect
import logging
import os.path
import re
import sys
import types

from docopt import docopt
import pkg_resources

__all__ = ('main',)

_COMMAND = os.path.basename(os.path.realpath(os.path.abspath(sys.argv[0])))
_DEFAULT_DOC = """
Usage: {command} <command> [<args>...]
       {command} (-h | --help)
       {command} (-V | --version)

Options:
  -h, --help           Display this help message and exit.
  -V, --version        Display the version and exit.
{{message}}
'{command} help -a' lists all available subcommands.
See '{command} help <command>' for more information on a specific command.
""".format(command=_COMMAND)

logging.getLogger().addHandler(logging.NullHandler())
_logger = logging.getLogger(__name__)
_subcommands = {}

def main():
    """Parse the command line options and launch the requested command.

    If the command is 'help' then print the help message for the subcommand; if
    no subcommand is given, print the standard help message.
    """
    _subcommands.update(_get_subcommands())
    dist_version = _get_dist_version()
    doc = _DEFAULT_DOC.format(message='')
    args = docopt(doc, version=dist_version, options_first=True)
    try:
        if args['<command>'] == 'help':
            subcommand = next(iter(args['<args>']), None)
            return _help(subcommand)
        else:
            argv = [args['<command>']] + args['<args>']
            return _run_command(argv)
    except (KeyboardInterrupt, EOFError):
        return "Cancelling at the user's request."
    except Exception as e:
        _logger.exception(e)
        return e

def _normalize(func, cli_args):
    """Alter the docopt args to be valid python names for func."""
    def _norm(k):
        """Normalize a single key."""
        if k.startswith('--'):
            k = k[2:]
        if k.startswith('-'):
            k = k[1:]
        if k.startswith('<') and k.endswith('>'):
            k = k[1:-1]
        return k.lower().replace('-', '_')

    if isinstance(func, types.FunctionType) or not hasattr(func, '__call__'):
        params = inspect.getargspec(func)[0]
    else:
        params = inspect.getargspec(func.__call__)[0][1:]
    args = {}
    for k, v in cli_args.items():
        nk = _norm(k)
        if nk in params:
            args[nk] = v
    return args

def _get_entry_point():
    """Return the current entry point."""
    mod_name = sys.modules[__name__].__name__
    for ep in pkg_resources.iter_entry_points(group='console_scripts'):
        if ep.name == _COMMAND and ep.module_name == mod_name:
            return ep

def _get_dist_version():
    """Return the version of the distribution that created this entry point."""
    entry_point = _get_entry_point()
    if entry_point and hasattr(entry_point.dist, 'version'):
        return str(entry_point.dist)

def _get_subcommands():
    """Return all subcommands for the current command."""
    regex = re.compile(r'{}:(?P<name>[^:]+)$'.format(_COMMAND))
    subcommands = {}
    for ep in pkg_resources.iter_entry_points(group='docopt_sub'):
        try:
            if ep.name == _COMMAND:
                subcommands[None] = ep.load()
            else:
                match = re.match(regex, ep.name)
                if match:
                    subcommands[match.group('name')] = ep.load()
        except ImportError as e:
            _logger.exception('Unable to load command. Skipping.', e)
    return subcommands

def _get_subcommand(name):
    """Return the function for the specified subcommand."""
    _logger.debug('Accessing subcommand "%s".', name)
    if name not in _subcommands:
        raise ValueError(
            '"{subcommand}" is not a {command} command. \'{command} help -a\' '
            'lists all available subcommands.'.format(
                command=_COMMAND, subcommand=name)
        )
    return _subcommands[name]

def _get_callable(subcommand, args):
    """Return a callable object from the subcommand."""
    _logger.debug(
        'Creating callable from subcommand "%s" with command line arguments: '
        '%s', subcommand.__name__, args)
    if isinstance(subcommand, types.ModuleType):
        _logger.debug('Subcommand is a module.')
        assert hasattr(subcommand, 'Command'), (
            'Module subcommand must have callable "Command" class definition.')
        subcommand = subcommand.Command
    if (isinstance(subcommand, types.ClassType) or
            isinstance(subcommand, types.TypeType)):
        try:
            return subcommand(**args)
        except TypeError:
            _logger.debug('Subcommand does not take arguments.')
            return subcommand()
    return subcommand

def _run_command(argv):
    """Run the command with the the given CLI options and exit.

    Command functions are expected to have a __doc__ string that is parseable
    by docopt. If the the function object has a 'validate' attribute, the
    arguments passed to the command will be validated before the command is
    called.

    Args:
        argv: The list of command line arguments supplied for a command. The
            first argument is expected to be the name of the command to be run.
            Note that this is different than the full arguments parsed by
            docopt for the entire program.

    Raises:
        ValueError: Raised if the user attempted to run an invalid command.
    """
    command_name = argv[0]
    _logger.info('Running command "%s" with args: %s', command_name, argv[1:])
    subcommand = _get_subcommand(command_name)
    _logger.debug('Parsing docstring:%s\nwith arguments %s.',
                  subcommand.__doc__, argv)
    args = docopt(subcommand.__doc__, argv=argv)
    call = _get_callable(subcommand, args)
    if hasattr(call, 'schema') and not hasattr(call, 'validate'):
        call.validate = call.schema.validate
    if hasattr(call, 'validate'):
        _logger.debug('Validating command arguments with "%s".', call.validate)
        args.update(call.validate(args))
    return call(**_normalize(call, args)) or 0

def _help(command):
    """Print out a help message and exit the program.

    Args:
        command: If a command value is supplied then print the help message for
            the command module if available. If the command is '-a' or '--all',
            then print the standard help message but with a full list of
            available commands.

    Raises:
        ValueError: Raised if the help message is requested for an invalid
            command or an unrecognized option is passed to help.
    """
    if not command:
        doc = _DEFAULT_DOC.format(message='')
    elif command in ('-a', '--all'):
        available_commands = (k for k in _subcommands.keys() if k)
        command_doc = '\nAvailable commands:\n{}\n'.format(
            '\n'.join('  {}'.format(c) for c in available_commands))
        doc = _DEFAULT_DOC.format(message=command_doc)
    elif command.startswith('-'):
        raise ValueError("Unrecognized option '{}'.".format(command))
    else:
        subcommand = _get_subcommand(command)
        doc = subcommand.__doc__
    docopt(doc, argv=('--help',))
<commit_msg>Switch exception logging to raise with a message where appropriate.<commit_after>#!/usr/bin/env python

"""An automatic command that handles subcommands in a git-like fashion."""

import inspect
import logging
import os.path
import re
import sys
import types

from docopt import docopt
import pkg_resources

__all__ = ('main',)

_COMMAND = os.path.basename(os.path.realpath(os.path.abspath(sys.argv[0])))
_DEFAULT_DOC = """
Usage: {command} <command> [<args>...]
       {command} (-h | --help)
       {command} (-V | --version)

Options:
  -h, --help           Display this help message and exit.
  -V, --version        Display the version and exit.
{{message}}
'{command} help -a' lists all available subcommands.
See '{command} help <command>' for more information on a specific command.
""".format(command=_COMMAND)

logging.getLogger().addHandler(logging.NullHandler())
_logger = logging.getLogger(__name__)
_subcommands = {}

def main():
    """Parse the command line options and launch the requested command.

    If the command is 'help' then print the help message for the subcommand; if
    no subcommand is given, print the standard help message.
    """
    _subcommands.update(_get_subcommands())
    dist_version = _get_dist_version()
    doc = _DEFAULT_DOC.format(message='')
    args = docopt(doc, version=dist_version, options_first=True)
    try:
        if args['<command>'] == 'help':
            subcommand = next(iter(args['<args>']), None)
            return _help(subcommand)
        else:
            argv = [args['<command>']] + args['<args>']
            return _run_command(argv)
    except (KeyboardInterrupt, EOFError):
        return "Cancelling at the user's request."
    except:
        _logger.exception('An unexpected error has occurred.')
        raise

def _normalize(func, cli_args):
    """Alter the docopt args to be valid python names for func."""
    def _norm(k):
        """Normalize a single key."""
        if k.startswith('--'):
            k = k[2:]
        if k.startswith('-'):
            k = k[1:]
        if k.startswith('<') and k.endswith('>'):
            k = k[1:-1]
        return k.lower().replace('-', '_')

    if isinstance(func, types.FunctionType) or not hasattr(func, '__call__'):
        params = inspect.getargspec(func)[0]
    else:
        params = inspect.getargspec(func.__call__)[0][1:]
    args = {}
    for k, v in cli_args.items():
        nk = _norm(k)
        if nk in params:
            args[nk] = v
    return args

def _get_entry_point():
    """Return the current entry point."""
    mod_name = sys.modules[__name__].__name__
    for ep in pkg_resources.iter_entry_points(group='console_scripts'):
        if ep.name == _COMMAND and ep.module_name == mod_name:
            return ep

def _get_dist_version():
    """Return the version of the distribution that created this entry point."""
    entry_point = _get_entry_point()
    if entry_point and hasattr(entry_point.dist, 'version'):
        return str(entry_point.dist)

def _get_subcommands():
    """Return all subcommands for the current command."""
    regex = re.compile(r'{}:(?P<name>[^:]+)$'.format(_COMMAND))
    subcommands = {}
    for ep in pkg_resources.iter_entry_points(group='docopt_sub'):
        try:
            if ep.name == _COMMAND:
                subcommands[None] = ep.load()
            else:
                match = re.match(regex, ep.name)
                if match:
                    subcommands[match.group('name')] = ep.load()
        except ImportError:
            _logger.exception('Unable to load command. Skipping.')
    return subcommands

def _get_subcommand(name):
    """Return the function for the specified subcommand."""
    _logger.debug('Accessing subcommand "%s".', name)
    if name not in _subcommands:
        raise ValueError(
            '"{subcommand}" is not a {command} command. \'{command} help -a\' '
            'lists all available subcommands.'.format(
                command=_COMMAND, subcommand=name)
        )
    return _subcommands[name]

def _get_callable(subcommand, args):
    """Return a callable object from the subcommand."""
    _logger.debug(
        'Creating callable from subcommand "%s" with command line arguments: '
        '%s', subcommand.__name__, args)
    if isinstance(subcommand, types.ModuleType):
        _logger.debug('Subcommand is a module.')
        assert hasattr(subcommand, 'Command'), (
            'Module subcommand must have callable "Command" class definition.')
        subcommand = subcommand.Command
    if (isinstance(subcommand, types.ClassType) or
            isinstance(subcommand, types.TypeType)):
        try:
            return subcommand(**args)
        except TypeError:
            _logger.debug('Subcommand does not take arguments.')
            return subcommand()
    return subcommand

def _run_command(argv):
    """Run the command with the the given CLI options and exit.

    Command functions are expected to have a __doc__ string that is parseable
    by docopt. If the the function object has a 'validate' attribute, the
    arguments passed to the command will be validated before the command is
    called.

    Args:
        argv: The list of command line arguments supplied for a command. The
            first argument is expected to be the name of the command to be run.
            Note that this is different than the full arguments parsed by
            docopt for the entire program.

    Raises:
        ValueError: Raised if the user attempted to run an invalid command.
    """
    command_name = argv[0]
    _logger.info('Running command "%s" with args: %s', command_name, argv[1:])
    subcommand = _get_subcommand(command_name)
    _logger.debug('Parsing docstring:%s\nwith arguments %s.',
                  subcommand.__doc__, argv)
    args = docopt(subcommand.__doc__, argv=argv)
    call = _get_callable(subcommand, args)
    if hasattr(call, 'schema') and not hasattr(call, 'validate'):
        call.validate = call.schema.validate
    if hasattr(call, 'validate'):
        _logger.debug('Validating command arguments with "%s".', call.validate)
        args.update(call.validate(args))
    return call(**_normalize(call, args)) or 0

def _help(command):
    """Print out a help message and exit the program.

    Args:
        command: If a command value is supplied then print the help message for
            the command module if available. If the command is '-a' or '--all',
            then print the standard help message but with a full list of
            available commands.

    Raises:
        ValueError: Raised if the help message is requested for an invalid
            command or an unrecognized option is passed to help.
    """
    if not command:
        doc = _DEFAULT_DOC.format(message='')
    elif command in ('-a', '--all'):
        available_commands = (k for k in _subcommands.keys() if k)
        command_doc = '\nAvailable commands:\n{}\n'.format(
            '\n'.join('  {}'.format(c) for c in available_commands))
        doc = _DEFAULT_DOC.format(message=command_doc)
    elif command.startswith('-'):
        raise ValueError("Unrecognized option '{}'.".format(command))
    else:
        subcommand = _get_subcommand(command)
        doc = subcommand.__doc__
    docopt(doc, argv=('--help',))
<|endoftext|>


<commit_before><commit_msg>Fixed to leaving the restore page and returning show status for all stages of the restore.<commit_after><|endoftext|>


<commit_before><commit_msg>add destination fields for invoice<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
#
# aws-instances.py
#
# A simple Python script that wraps the AWS CLI (https://aws.amazon.com/cli/)
# and generates a table view of instantiated EC2 instances running on AWS.
#
import json
import subprocess
import sys
import argparse

# FUNCTIONS
#
#

def columns_layout(extended=False):
    if extended:
        return "{0: <15} {1: <16} {2: <16} {3: <50} {4: <45} {5: <15} {6}"
    else:
        return "{0: <16} {1: <17} {2: <42} {3: <10} {4}"

def header(extended=False):
    if extended:
        return columns_layout(extended).format(
            'Instance Type',
            'PublicIpAddress',
            'PrivateIpAddress',
            'PublicDnsName',
            'PrivateDnsName',
            'State',
            'Tag name'
        )
    else:
        return columns_layout(extended).format(
            'PublicIpAddress',
            'PrivateIpAddress',
            'PrivateDnsName',
            'State',
            'Tag name'
        )

def to_table(extended=False):
    nodes = []
    nodes.append(header(extended))

    for reservations in desc['Reservations']:
        for instances in reservations['Instances']:
            public_ip_address = instances.get('PublicIpAddress', "-") or "-"
            private_ip_address = instances.get('PrivateIpAddress', "-") or "-"
            public_dns_name = instances.get('PublicDnsName', "-") or "-"
            private_dns_name = instances.get('PrivateDnsName', "-") or "-"

            tag_name = 'UNKNOWN'
            if 'Tags' in instances.keys():
                for tags in instances['Tags']:
                    key = tags['Key']
                    value = tags['Value']
                    if key == 'Name':
                        tag_name = "{}\t".format(value)

            if extended:
                nodes.append(columns_layout(extended).format(
                    instances['InstanceType'],
                    public_ip_address,
                    private_ip_address,
                    public_dns_name,
                    private_dns_name,
                    instances['State']['Name'],
                    tag_name
                ))
            else:
                nodes.append(columns_layout(extended).format(
                    public_ip_address,
                    private_ip_address,
                    private_dns_name,
                    instances['State']['Name'],
                    tag_name
                ))
    return nodes

def awsdescribe():
    """Run aws ec2 describe-instances"""
    p = subprocess.Popen(["aws", "ec2", "describe-instances"], stdout=subprocess.PIPE)
    (output, err) = p.communicate()
    return output

# MAIN
#
#

cmdargs = str(sys.argv)

desc = {}
try:
    desc = json.loads(awsdescribe())
except ValueError as ve:
    print "Oops! Something went wrong: {}".format(ve)
    sys.exit(1)

parser = argparse.ArgumentParser()
parser.add_argument('-e','--extended', help='Extended output', action='store_const', const=True)
args = parser.parse_args()

for line in to_table(args.extended == True):
    print line
<commit_msg>feat: remove PublicIpAddress from short version output<commit_after>#!/usr/bin/env python
#
# aws-instances.py
#
# A simple Python script that wraps the AWS CLI (https://aws.amazon.com/cli/)
# and generates a table view of instantiated EC2 instances running on AWS.
#
import json
import subprocess
import sys
import argparse

# FUNCTIONS
#
#

def columns_layout(extended=False):
    if extended:
        return "{0: <15} {1: <16} {2: <16} {3: <50} {4: <45} {5: <15} {6}"
    else:
        return "{0: <17} {1: <42} {2: <10} {3}"

def header(extended=False):
    if extended:
        return columns_layout(extended).format(
            'Instance Type',
            'PublicIpAddress',
            'PrivateIpAddress',
            'PublicDnsName',
            'PrivateDnsName',
            'State',
            'Tag name'
        )
    else:
        return columns_layout(extended).format(
            'PrivateIpAddress',
            'PrivateDnsName',
            'State',
            'Tag name'
        )

def to_table(extended=False):
    nodes = []
    nodes.append(header(extended))

    for reservations in desc['Reservations']:
        for instances in reservations['Instances']:
            public_ip_address = instances.get('PublicIpAddress', "-") or "-"
            private_ip_address = instances.get('PrivateIpAddress', "-") or "-"
            public_dns_name = instances.get('PublicDnsName', "-") or "-"
            private_dns_name = instances.get('PrivateDnsName', "-") or "-"

            tag_name = 'UNKNOWN'
            if 'Tags' in instances.keys():
                for tags in instances['Tags']:
                    key = tags['Key']
                    value = tags['Value']
                    if key == 'Name':
                        tag_name = "{}\t".format(value)

            if extended:
                nodes.append(columns_layout(extended).format(
                    instances['InstanceType'],
                    public_ip_address,
                    private_ip_address,
                    public_dns_name,
                    private_dns_name,
                    instances['State']['Name'],
                    tag_name
                ))
            else:
                nodes.append(columns_layout(extended).format(
                    private_ip_address,
                    private_dns_name,
                    instances['State']['Name'],
                    tag_name
                ))
    return nodes

def awsdescribe():
    """Run aws ec2 describe-instances"""
    p = subprocess.Popen(["aws", "ec2", "describe-instances"], stdout=subprocess.PIPE)
    (output, err) = p.communicate()
    return output

# MAIN
#
#

cmdargs = str(sys.argv)

desc = {}
try:
    desc = json.loads(awsdescribe())
except ValueError as ve:
    print "Oops! Something went wrong: {}".format(ve)
    sys.exit(1)

parser = argparse.ArgumentParser()
parser.add_argument('-e','--extended', help='Extended output', action='store_const', const=True)
args = parser.parse_args()

for line in to_table(args.extended == True):
    print line
<|endoftext|>


<commit_before># Configuration module

#
## tmp note: you should add logger module and modify this module
#
# tmp change

from ConfigParser import SafeConfigParser
import os
from traceback import format_exc

program_name = "migmig"
# config file short path (hard coded !)
cfg_short_path = "~/." + program_name + ".ini"
server_address = "localhost"
server_port = "50001"

class Configuration():

	def __init__(self, logger, user_options):
		# constant variables
		self.OK = '200'
		self.DONE = '199'
		self.BAD_URL = '98'
		self.BAD_IDENTIFIER = '97'
		self.NOT_FOUND = '404'
		self.RANGE_NOT_SUPPORTED = '198'
		self.SOMETHING = '197'

		self.logger = logger.get_logger()
		self.user_options = user_options
		self.parser = SafeConfigParser()
		self.cfg_path = self.validate_path(cfg_short_path)

		if not self.cfg_path:
			# create config file and initate configurations
			self.cfg_path = os.path.expanduser(cfg_short_path)
			self.logger.info('Configuration path doesnt exist, creating new one in %s', self.cfg_path)
			self.initate()

		self.logger.info('Initating Safe Config Parser. ')
		self.parser.read(self.cfg_path)

		self.check_download_path()

	def initate(self):
		# Sections
		self.parser.add_section("Setting")
		self.parser.add_section("Client")

		# Options
		self.reset_setting()
		self.reset_client()

		# write settings to ini file
		# self.write()			# reset_*() will write to the files, no need for this line

	def get(self, name):
		'''
			This method looks for given name in all sections,
			returns the first value that is matched.

			User's options priority is higher, so if an option
			is specified by user, that option returns.
		'''
		self.logger.debug('Getting %s from config parser.' % name)
		try:
			if name in self.user_options:
				return self.user_options[name]

			for section in self.parser.sections():
				for item, val in self.parser.items(section):
					if name == item:
						return val
		except:
			# TO-DO: Log, maybe this option doesn not exist at all!
			self.logger.warning('%s doesn\'t exist in config file.' % name)
			return None

	def set(self, **kwargs):
		'''
			This only works for "Client" section.
		'''
		self.logger.debug('Writing new options to config file. options are: (%s)' % str(kwargs))
		try:
			for key, value in kwargs.items():
				self.parser.set('Client', key, value)

			self.write()
			return True
		except:
			self.logger.error(format_exc().split('\n')[-2])
			self.logger.error('Cannot set (or write) the key, value.')
			return False

	def write(self):
		'''
			Write (synchronize) the parser object to .ini file !
		'''
		with open(self.cfg_path, "wb") as tmp:
				self.parser.write(tmp)

	def get_server(self):
		addr = self.parser.get('Setting','server_address')
		port = self.parser.get('Setting','server_port')
		return "http://" + addr + ':' + port

	def check_download_path(self):
		'''
			If download_path does not exist, create !
		'''
		d_path = self.parser.get('Setting', 'download_path')
		if not os.path.exists(d_path):
			self.logger.info('download path (%s) doesnt exist, creating the path.' % d_path)
			try:
				os.makedirs(d_path)
			except:
				# maybe permission denied ?
				self.logger.critical('(%s) cannot be created.' % (d_path))
				raise
		return True

	def validate_path(self, path):
		'''
			Check whether path exists or not
		'''
		path = os.path.expanduser(path)
		if os.path.exists(path):
			return path
		return None

	def reset_client(self):
		self.parser.set('Client', 'identifier', 'None')
		self.parser.set('Client', 'url', 'None')
		self.parser.set('Client', 'client_id', 'None')
		self.parser.set('Client', 'latest_chunk', 'None')
		# update the config file
		self.write()

	def reset_setting(self):
		self.parser.set("Setting", "download_path", os.path.expanduser("~/Downloads/" + program_name))
		self.parser.set("Setting", "default_merge_path", os.path.expanduser("~/Downloads/" + program_name + "/merged"))
		self.parser.set("Setting", "max-conn", "6")
		self.parser.set("Setting", "retries", "3")

		self.parser.set('Setting', 'server_address', server_address)
		self.parser.set('Setting', 'server_port', server_port)
		# update the file
		self.write()
<commit_msg>Add new method: reset_client()<commit_after># Configuration module
#
## tmp note: you should add logger module and modify this module
#

from ConfigParser import SafeConfigParser
import os
from traceback import format_exc

program_name = "migmig"
# config file short path (hard coded !)
cfg_short_path = "~/." + program_name + ".ini"
server_address = "localhost"
server_port = "50001"

class Configuration():

	def __init__(self, logger, user_options):
		# constant variables
		self.OK = '200'
		self.DONE = '199'
		self.BAD_URL = '98'
		self.BAD_IDENTIFIER = '97'
		self.NOT_FOUND = '404'
		self.RANGE_NOT_SUPPORTED = '198'
		self.SOMETHING = '197'

		self.logger = logger.get_logger()
		self.user_options = user_options
		self.parser = SafeConfigParser()
		self.cfg_path = self.validate_path(cfg_short_path)

		if not self.cfg_path:
			# create config file and initate configurations
			self.cfg_path = os.path.expanduser(cfg_short_path)
			self.logger.info('Configuration path doesnt exist, creating new one in %s', self.cfg_path)
			self.initate()

		self.logger.info('Initating Safe Config Parser. ')
		self.parser.read(self.cfg_path)

		self.check_download_path()

	def initate(self):
		# Sections
		self.parser.add_section("Setting")
		self.parser.add_section("Client")

		# Options
		self.reset_setting()

		# write settings to ini file
		# self.write()			# no need for this line

	def get(self, name):
		'''
			This method looks for given name in all sections,
			returns the first value that is matched.

			User's options priority is higher, so if an option
			is specified by user, that option returns.
		'''
		self.logger.debug('Getting %s from config parser.' % name)
		try:
			if name in self.user_options:
				return self.user_options[name]

			for section in self.parser.sections():
				for item, val in self.parser.items(section):
					if name == item:
						return val
		except:
			# TO-DO: Log, maybe this option doesn not exist at all!
			self.logger.warning('%s doesn\'t exist in config file.' % name)
			return None

	def set(self, **kwargs):
		'''
			This only works for "Client" section.
		'''
		self.logger.debug('Writing new options to config file. options are: (%s)' % str(kwargs))
		try:
			for key, value in kwargs.items():
				self.parser.set('Client', key, value)

			self.write()
			return True
		except:
			self.logger.error(format_exc().split('\n')[-2])
			self.logger.error('Cannot set (or write) the key, value.')
			return False

	def write(self):
		'''
			Write (synchronize) the parser object to .ini file !
		'''
		with open(self.cfg_path, "wb") as tmp:
				self.parser.write(tmp)

	def get_server(self):
		addr = self.parser.get('Setting','server_address')
		port = self.parser.get('Setting','server_port')
		return "http://" + addr + ':' + port

	def check_download_path(self):
		'''
			If download_path does not exist, create !
		'''
		d_path = self.parser.get('Setting', 'download_path')
		if not os.path.exists(d_path):
			self.logger.info('download path (%s) doesnt exist, creating the path.' % d_path)
			try:
				os.makedirs(d_path)
			except:
				# maybe permission denied ?
				self.logger.critical('(%s) cannot be created.' % (d_path))
				raise
		return True

	def validate_path(self, path):
		'''
			Check whether path exists or not
		'''
		path = os.path.expanduser(path)
		if os.path.exists(path):
			return path
		return None

	def reset_client(self):
		# removes all the options in "Client" section
		self.parser.remove_section('Client')
		self.parser.add_section('Client')
		self.write()

	def reset_setting(self):
		self.parser.set("Setting", "download_path", os.path.expanduser("~/Downloads/" + program_name))
		self.parser.set("Setting", "default_merge_path", os.path.expanduser("~/Downloads/" + program_name + "/merged"))
		self.parser.set("Setting", "max-conn", "6")
		self.parser.set("Setting", "retries", "3")

		self.parser.set('Setting', 'server_address', server_address)
		self.parser.set('Setting', 'server_port', server_port)
		# update the file
		self.write()
<|endoftext|>


<commit_before>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on October 5, 2014 (fork of Examples/Basic/tutorial1.py)

@author: Bradley Wogsland
@contact: bradley@wogsland.org
@summary: Solution to Homework 1
'''

# QSTK Imports
import QSTK.qstkutil.qsdateutil as du
import QSTK.qstkutil.tsutil as tsu
import QSTK.qstkutil.DataAccess as da

# Third Party Imports
import datetime as dt
import matplotlib.pyplot as plt
import pandas as pd
import math

# Example call of function:
# vol, daily_ret, sharpe, cum_ret = simulate(startdate, enddate, ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
# The function should return:
#   Standard deviation of daily returns of the total portfolio
#   Average daily return of the total portfolio
#   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
#   Cumulative return of the total portfolio
def simulate(startdate, endate, symbols, allocations):
    ''' Main Function'''
    print "In the simulate function."

    # We need closing prices so the timestamp should be hours=16.
    timeofday = dt.timedelta(hours=16)

    # Get a list of trading days between the start and the end.
    ldt_timestamps = du.getNYSEdays(startdate, endate, timeofday)

    # Creating an object of the dataaccess class with Yahoo as the source.
    c_dataobj = da.DataAccess('Yahoo')

    # Keys to be read from the data, it is good to read everything in one go.
    #ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']
    ls_keys = ['close']

    # Reading the data, now d_data is a dictionary with the keys above.
    # Timestamps and symbols are the ones that were specified before.
    ldf_data = c_dataobj.get_data(ldt_timestamps, symbols, ls_keys)
    d_data = dict(zip(ls_keys, ldf_data))
    #print d_data

    # Filling the data for NAN
    for s_key in ls_keys:
        d_data[s_key] = d_data[s_key].fillna(method='ffill')
        d_data[s_key] = d_data[s_key].fillna(method='bfill')
        d_data[s_key] = d_data[s_key].fillna(1.0)

    # Getting the numpy ndarray of close prices.
    na_price = d_data['close'].values

    # Normalizing the prices to start at 1 and see relative returns
    na_normalized_price = na_price / na_price[0, :]

    # Copy the normalized prices to a new ndarry to find returns.
    #na_rets = na_normalized_price.copy()
    na_rets = na_price.copy()
    #na_rets = na_price.copy() * allocations
    #na_rets = na_normalized_price.copy() * allocations
    #print na_rets

    # Calculate the daily returns of the prices. (Inplace calculation)
    # returnize0 works on ndarray and not dataframes.
    tsu.returnize0(na_rets)

    # apply allocations
    na_rets = na_rets * allocations
    na_rets = na_rets.sum(axis=1)
    print na_rets

    #   Standard deviation of daily returns of the total portfolio
    std = na_rets.std()

    #   Average daily return of the total portfolio
    avg = na_rets.mean()

    #   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
    sh = math.sqrt(252)*(avg/std)

    #   Cumulative return of the total portfolio
    cum = na_rets.sum()

    return std,avg,sh,cum

# Try executing the function to test
vol, daily_ret, sharpe, cum_ret = simulate(dt.datetime(2011,1,1), dt.datetime(2011,12,31), ['AAPL','GLD','GOOG','XOM'], [0.4,0.4,0.0,0.2])
print "params dt.datetime(2011,1,1), dt.datetime(2011,12,31), ['AAPL','GLD','GOOG','XOM'], [0.4,0.4,0.0,0.2]"
print "vol = " + str(vol)
print "Volatility (stdev of daily returns):  0.0101467067654"
print "daily_ret = " + str(daily_ret)
print "Average Daily Return:  0.000657261102001"
print "sharpe = " + str(sharpe)
print "Sharpe Ratio: 1.02828403099"
print "cum_ret = " + str(cum_ret)
print "Cumulative Return:  1.16487261965"

vol, daily_ret, sharpe, cum_ret = simulate(dt.datetime(2010,1,1), dt.datetime(2010,12,31), ['AXP', 'HPQ', 'IBM', 'HNZ'], [0.0, 0.0, 0.0, 1.0])
print "params dt.datetime(2010,1,1), dt.datetime(2010,12,31), ['AXP', 'HPQ', 'IBM', 'HNZ'], [0.0, 0.0, 0.0, 1.0]"
print "vol = " + str(vol)
print "Volatility (stdev of daily returns):  0.00924299255937"
print "daily_ret = " + str(daily_ret)
print "Average Daily Return:  0.000756285585593"
print "sharpe = " + str(sharpe)
print "Sharpe Ratio: 1.29889334008"
print "cum_ret = " + str(cum_ret)
print "Cumulative Return:  1.1960583568"
<commit_msg>testing<commit_after>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on October 5, 2014 (fork of Examples/Basic/tutorial1.py)

@author: Bradley Wogsland
@contact: bradley@wogsland.org
@summary: Solution to Homework 1
'''

# QSTK Imports
import QSTK.qstkutil.qsdateutil as du
import QSTK.qstkutil.tsutil as tsu
import QSTK.qstkutil.DataAccess as da

# Third Party Imports
import datetime as dt
import matplotlib.pyplot as plt
import pandas as pd
import math

# Example call of function:
# vol, daily_ret, sharpe, cum_ret = simulate(startdate, enddate, ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
# The function should return:
#   Standard deviation of daily returns of the total portfolio
#   Average daily return of the total portfolio
#   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
#   Cumulative return of the total portfolio
def simulate(startdate, endate, symbols, allocations):
    ''' Main Function'''
    print "In the simulate function."

    # We need closing prices so the timestamp should be hours=16.
    timeofday = dt.timedelta(hours=16)

    # Get a list of trading days between the start and the end.
    ldt_timestamps = du.getNYSEdays(startdate, endate, timeofday)

    # Creating an object of the dataaccess class with Yahoo as the source.
    c_dataobj = da.DataAccess('Yahoo')

    # Keys to be read from the data, it is good to read everything in one go.
    #ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']
    ls_keys = ['close']

    # Reading the data, now d_data is a dictionary with the keys above.
    # Timestamps and symbols are the ones that were specified before.
    ldf_data = c_dataobj.get_data(ldt_timestamps, symbols, ls_keys)
    d_data = dict(zip(ls_keys, ldf_data))
    #print d_data

    # Filling the data for NAN
    for s_key in ls_keys:
        d_data[s_key] = d_data[s_key].fillna(method='ffill')
        d_data[s_key] = d_data[s_key].fillna(method='bfill')
        d_data[s_key] = d_data[s_key].fillna(1.0)

    # Getting the numpy ndarray of close prices.
    na_price = d_data['close'].values

    # Normalizing the prices to start at 1 and see relative returns
    na_normalized_price = na_price / na_price[0, :]

    # Copy the normalized prices to a new ndarry to find returns.
    #na_rets = na_normalized_price.copy()
    na_rets = na_price.copy()
    #na_rets = na_price.copy() * allocations
    #na_rets = na_normalized_price.copy() * allocations
    #print na_rets

    # Calculate the daily returns of the prices. (Inplace calculation)
    # returnize0 works on ndarray and not dataframes.
    tsu.returnize1(na_rets)

    # apply allocations
    na_rets = na_rets * allocations
    na_rets = na_rets.sum(axis=1)
    #print na_rets

    #   Standard deviation of daily returns of the total portfolio
    std = na_rets.std()

    #   Average daily return of the total portfolio
    avg = na_rets.mean()

    #   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
    sh = math.sqrt(252)*(avg/std)

    #   Cumulative return of the total portfolio
    cum = na_rets.sum()

    return std,avg,sh,cum

# Try executing the function to test
vol, daily_ret, sharpe, cum_ret = simulate(dt.datetime(2011,1,1), dt.datetime(2011,12,31), ['AAPL','GLD','GOOG','XOM'], [0.4,0.4,0.0,0.2])
print "params dt.datetime(2011,1,1), dt.datetime(2011,12,31), ['AAPL','GLD','GOOG','XOM'], [0.4,0.4,0.0,0.2]"
print "vol = " + str(vol)
print "Volatility (stdev of daily returns):  0.0101467067654"
print "daily_ret = " + str(daily_ret)
print "Average Daily Return:  0.000657261102001"
print "sharpe = " + str(sharpe)
print "Sharpe Ratio: 1.02828403099"
print "cum_ret = " + str(cum_ret)
print "Cumulative Return:  1.16487261965"

vol, daily_ret, sharpe, cum_ret = simulate(dt.datetime(2010,1,1), dt.datetime(2010,12,31), ['AXP', 'HPQ', 'IBM', 'HNZ'], [0.0, 0.0, 0.0, 1.0])
print "params dt.datetime(2010,1,1), dt.datetime(2010,12,31), ['AXP', 'HPQ', 'IBM', 'HNZ'], [0.0, 0.0, 0.0, 1.0]"
print "vol = " + str(vol)
print "Volatility (stdev of daily returns):  0.00924299255937"
print "daily_ret = " + str(daily_ret)
print "Average Daily Return:  0.000756285585593"
print "sharpe = " + str(sharpe)
print "Sharpe Ratio: 1.29889334008"
print "cum_ret = " + str(cum_ret)
print "Cumulative Return:  1.1960583568"
<|endoftext|>