<commit_before>"""
Monitor thrash
"""
import logging
import contextlib
import ceph_manager
import random
import time
import gevent
import json
import math
from teuthology import misc as teuthology

log = logging.getLogger(__name__)

def _get_mons(ctx):
    """
    Get monitor names from the context value.
    """
    mons = [f[len('mon.'):] for f in teuthology.get_mon_names(ctx)]
    return mons

class MonitorThrasher:
    """
    How it works::

    - pick a monitor
    - kill it
    - wait for quorum to be formed
    - sleep for 'revive_delay' seconds
    - revive monitor
    - wait for quorum to be formed
    - sleep for 'thrash_delay' seconds

    Options::

    seed                Seed to use on the RNG to reproduce a previous
                        behaviour (default: None; i.e., not set)
    revive_delay        Number of seconds to wait before reviving
                        the monitor (default: 10)
    thrash_delay        Number of seconds to wait in-between
                        test iterations (default: 0)
    thrash_store        Thrash monitor store before killing the monitor being thrashed (default: False)
    thrash_store_probability  Probability of thrashing a monitor's store
                              (default: 50)
    thrash_many         Thrash multiple monitors instead of just one. If
                        'maintain-quorum' is set to False, then we will
                        thrash up to as many monitors as there are
                        available. (default: False)
    maintain_quorum     Always maintain quorum, taking care on how many
                        monitors we kill during the thrashing. If we
                        happen to only have one or two monitors configured,
                        if this option is set to True, then we won't run
                        this task as we cannot guarantee maintenance of
                        quorum. Setting it to false however would allow the
                        task to run with as many as just one single monitor.
                        (default: True)
    freeze_mon_probability: how often to freeze the mon instead of killing it,
                        in % (default: 0)
    freeze_mon_duration: how many seconds to freeze the mon (default: 15)
    scrub               Scrub after each iteration (default: True)

    Note: if 'store-thrash' is set to True, then 'maintain-quorum' must also
          be set to True.

    For example::

    tasks:
    - ceph:
    - mon_thrash:
        revive_delay: 20
        thrash_delay: 1
        thrash_store: true
        thrash_store_probability: 40
        seed: 31337
        maintain_quorum: true
        thrash_many: true
    - ceph-fuse:
    - workunit:
        clients:
          all:
            - mon/workloadgen.sh
    """
    def __init__(self, ctx, manager, config, logger):
        self.ctx = ctx
        self.manager = manager
        self.manager.wait_for_clean()

        self.stopping = False
        self.logger = logger
        self.config = config

        if self.config is None:
            self.config = dict()

        """ Test reproducibility """
        self.random_seed = self.config.get('seed', None)

        if self.random_seed is None:
            self.random_seed = int(time.time())

        self.rng = random.Random()
        self.rng.seed(int(self.random_seed))

        """ Monitor thrashing """
        self.revive_delay = float(self.config.get('revive_delay', 10.0))
        self.thrash_delay = float(self.config.get('thrash_delay', 0.0))

        self.thrash_many = self.config.get('thrash_many', False)
        self.maintain_quorum = self.config.get('maintain_quorum', True)

        self.scrub = self.config.get('scrub', True)

        self.freeze_mon_probability = float(self.config.get('freeze_mon_probability', 10))
        self.freeze_mon_duration = float(self.config.get('freeze_mon_duration', 15.0))

        assert self.max_killable() > 0, \
            'Unable to kill at least one monitor with the current config.'

        """ Store thrashing """
        self.store_thrash = self.config.get('store_thrash', False)
        self.store_thrash_probability = int(
            self.config.get('store_thrash_probability', 50))
        if self.store_thrash:
            assert self.store_thrash_probability > 0, \
                'store_thrash is set, probability must be > 0'
            assert self.maintain_quorum, \
                'store_thrash = true must imply maintain_quorum = true'

        self.thread = gevent.spawn(self.do_thrash)

    def log(self, x):
        """
        locally log info messages
        """
        self.logger.info(x)

    def do_join(self):
        """
        Break out of this processes thrashing loop.
        """
        self.stopping = True
        self.thread.get()

    def should_thrash_store(self):
        """
        If allowed, indicate that we should thrash a certain percentage of
        the time as determined by the store_thrash_probability value.
        """
        if not self.store_thrash:
            return False
        return self.rng.randrange(0, 101) < self.store_thrash_probability

    def thrash_store(self, mon):
        """
        Thrash the monitor specified.
        :param mon: monitor to thrash
        """
        addr = self.ctx.ceph.conf['mon.%s' % mon]['mon addr']
        self.log('thrashing mon.{id}@{addr} store'.format(id=mon, addr=addr))
        out = self.manager.raw_cluster_cmd('-m', addr, 'sync', 'force')
        j = json.loads(out)
        assert j['ret'] == 0, \
            'error forcing store sync on mon.{id}:\n{ret}'.format(
                id=mon,ret=out)

    def should_freeze_mon(self):
        """
        Indicate that we should freeze a certain percentago of the time
        as determined by the freeze_mon_probability value.
        """
        return self.rng.randrange(0, 101) < self.freeze_mon_probability

    def freeze_mon(self, mon):
        """
        Send STOP signal to freeze the monitor.
        """
        log.info('Sending STOP to mon %s', mon)
        self.manager.signal_mon(mon, 19)  # STOP

    def unfreeze_mon(self, mon):
        """
        Send CONT signal to unfreeze the monitor.
        """
        log.info('Sending CONT to mon %s', mon)
        self.manager.signal_mon(mon, 18)  # CONT

    def kill_mon(self, mon):
        """
        Kill the monitor specified
        """
        self.log('killing mon.{id}'.format(id=mon))
        self.manager.kill_mon(mon)

    def revive_mon(self, mon):
        """
        Revive the monitor specified
        """
        self.log('killing mon.{id}'.format(id=mon))
        self.log('reviving mon.{id}'.format(id=mon))
        self.manager.revive_mon(mon)

    def max_killable(self):
        """
        Return the maximum number of monitors we can kill.
        """
        m = len(_get_mons(self.ctx))
        if self.maintain_quorum:
            return max(math.ceil(m/2.0)-1, 0)
        else:
            return m

    def do_thrash(self):
        """
        Cotinuously loop and thrash the monitors.
        """
        self.log('start thrashing')
        self.log('seed: {s}, revive delay: {r}, thrash delay: {t} '\
                   'thrash many: {tm}, maintain quorum: {mq} '\
                   'store thrash: {st}, probability: {stp} '\
                   'freeze mon: prob {fp} duration {fd}'.format(
                s=self.random_seed,r=self.revive_delay,t=self.thrash_delay,
                tm=self.thrash_many, mq=self.maintain_quorum,
                st=self.store_thrash,stp=self.store_thrash_probability,
                fp=self.freeze_mon_probability,fd=self.freeze_mon_duration,
                ))

        while not self.stopping:
            mons = _get_mons(self.ctx)
            self.manager.wait_for_mon_quorum_size(len(mons))
            self.log('making sure all monitors are in the quorum')
            for m in mons:
                s = self.manager.get_mon_status(m)
                assert s['state'] == 'leader' or s['state'] == 'peon'
                assert len(s['quorum']) == len(mons)

            kill_up_to = self.rng.randrange(1, self.max_killable()+1)
            mons_to_kill = self.rng.sample(mons, kill_up_to)
            self.log('monitors to thrash: {m}'.format(m=mons_to_kill))

            mons_to_freeze = []
            for mon in mons:
                if mon in mons_to_kill:
                    continue
                if self.should_freeze_mon():
                    mons_to_freeze.append(mon)
            self.log('monitors to freeze: {m}'.format(m=mons_to_freeze))

            for mon in mons_to_kill:
                self.log('thrashing mon.{m}'.format(m=mon))

                """ we only thrash stores if we are maintaining quorum """
                if self.should_thrash_store() and self.maintain_quorum:
                    self.thrash_store(mon)

                self.kill_mon(mon)

            if mons_to_freeze:
                for mon in mons_to_freeze:
                    self.freeze_mon(mon)
                self.log('waiting for {delay} secs to unfreeze mons'.format(
                    delay=self.freeze_mon_duration))
                time.sleep(self.freeze_mon_duration)
                for mon in mons_to_freeze:
                    self.unfreeze_mon(mon)

            if self.maintain_quorum:
                self.manager.wait_for_mon_quorum_size(len(mons)-len(mons_to_kill))
                for m in mons:
                    if m in mons_to_kill:
                        continue
                    s = self.manager.get_mon_status(m)
                    assert s['state'] == 'leader' or s['state'] == 'peon'
                    assert len(s['quorum']) == len(mons)-len(mons_to_kill)

            self.log('waiting for {delay} secs before reviving monitors'.format(
                delay=self.revive_delay))
            time.sleep(self.revive_delay)

            for mon in mons_to_kill:
                self.revive_mon(mon)
            # do more freezes
            if mons_to_freeze:
                for mon in mons_to_freeze:
                    self.freeze_mon(mon)
                self.log('waiting for {delay} secs to unfreeze mons'.format(
                    delay=self.freeze_mon_duration))
                time.sleep(self.freeze_mon_duration)
                for mon in mons_to_freeze:
                    self.unfreeze_mon(mon)

            self.manager.wait_for_mon_quorum_size(len(mons))
            for m in mons:
                s = self.manager.get_mon_status(m)
                assert s['state'] == 'leader' or s['state'] == 'peon'
                assert len(s['quorum']) == len(mons)

            if self.scrub:
                self.log('triggering scrub')
                try:
                    self.manager.raw_cluster_cmd('scrub')
                except Exception:
                    pass

            if self.thrash_delay > 0.0:
                self.log('waiting for {delay} secs before continuing thrashing'.format(
                    delay=self.thrash_delay))
                time.sleep(self.thrash_delay)

@contextlib.contextmanager
def task(ctx, config):
    """
    Stress test the monitor by thrashing them while another task/workunit
    is running.

    Please refer to MonitorThrasher class for further information on the
    available options.
    """
    if config is None:
        config = {}
    assert isinstance(config, dict), \
        'mon_thrash task only accepts a dict for configuration'
    assert len(_get_mons(ctx)) > 2, \
        'mon_thrash task requires at least 3 monitors'
    log.info('Beginning mon_thrash...')
    first_mon = teuthology.get_first_mon(ctx, config)
    (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
    manager = ceph_manager.CephManager(
        mon,
        ctx=ctx,
        logger=log.getChild('ceph_manager'),
        )
    thrash_proc = MonitorThrasher(ctx,
        manager, config,
        logger=log.getChild('mon_thrasher'))
    try:
        log.debug('Yielding')
        yield
    finally:
        log.info('joining mon_thrasher')
        thrash_proc.do_join()
        mons = _get_mons(ctx)
        manager.wait_for_mon_quorum_size(len(mons))
<commit_msg>When ignoring a raised exception, at least log it<commit_after>"""
Monitor thrash
"""
import logging
import contextlib
import ceph_manager
import random
import time
import gevent
import json
import math
from teuthology import misc as teuthology

log = logging.getLogger(__name__)

def _get_mons(ctx):
    """
    Get monitor names from the context value.
    """
    mons = [f[len('mon.'):] for f in teuthology.get_mon_names(ctx)]
    return mons

class MonitorThrasher:
    """
    How it works::

    - pick a monitor
    - kill it
    - wait for quorum to be formed
    - sleep for 'revive_delay' seconds
    - revive monitor
    - wait for quorum to be formed
    - sleep for 'thrash_delay' seconds

    Options::

    seed                Seed to use on the RNG to reproduce a previous
                        behaviour (default: None; i.e., not set)
    revive_delay        Number of seconds to wait before reviving
                        the monitor (default: 10)
    thrash_delay        Number of seconds to wait in-between
                        test iterations (default: 0)
    thrash_store        Thrash monitor store before killing the monitor being thrashed (default: False)
    thrash_store_probability  Probability of thrashing a monitor's store
                              (default: 50)
    thrash_many         Thrash multiple monitors instead of just one. If
                        'maintain-quorum' is set to False, then we will
                        thrash up to as many monitors as there are
                        available. (default: False)
    maintain_quorum     Always maintain quorum, taking care on how many
                        monitors we kill during the thrashing. If we
                        happen to only have one or two monitors configured,
                        if this option is set to True, then we won't run
                        this task as we cannot guarantee maintenance of
                        quorum. Setting it to false however would allow the
                        task to run with as many as just one single monitor.
                        (default: True)
    freeze_mon_probability: how often to freeze the mon instead of killing it,
                        in % (default: 0)
    freeze_mon_duration: how many seconds to freeze the mon (default: 15)
    scrub               Scrub after each iteration (default: True)

    Note: if 'store-thrash' is set to True, then 'maintain-quorum' must also
          be set to True.

    For example::

    tasks:
    - ceph:
    - mon_thrash:
        revive_delay: 20
        thrash_delay: 1
        thrash_store: true
        thrash_store_probability: 40
        seed: 31337
        maintain_quorum: true
        thrash_many: true
    - ceph-fuse:
    - workunit:
        clients:
          all:
            - mon/workloadgen.sh
    """
    def __init__(self, ctx, manager, config, logger):
        self.ctx = ctx
        self.manager = manager
        self.manager.wait_for_clean()

        self.stopping = False
        self.logger = logger
        self.config = config

        if self.config is None:
            self.config = dict()

        """ Test reproducibility """
        self.random_seed = self.config.get('seed', None)

        if self.random_seed is None:
            self.random_seed = int(time.time())

        self.rng = random.Random()
        self.rng.seed(int(self.random_seed))

        """ Monitor thrashing """
        self.revive_delay = float(self.config.get('revive_delay', 10.0))
        self.thrash_delay = float(self.config.get('thrash_delay', 0.0))

        self.thrash_many = self.config.get('thrash_many', False)
        self.maintain_quorum = self.config.get('maintain_quorum', True)

        self.scrub = self.config.get('scrub', True)

        self.freeze_mon_probability = float(self.config.get('freeze_mon_probability', 10))
        self.freeze_mon_duration = float(self.config.get('freeze_mon_duration', 15.0))

        assert self.max_killable() > 0, \
            'Unable to kill at least one monitor with the current config.'

        """ Store thrashing """
        self.store_thrash = self.config.get('store_thrash', False)
        self.store_thrash_probability = int(
            self.config.get('store_thrash_probability', 50))
        if self.store_thrash:
            assert self.store_thrash_probability > 0, \
                'store_thrash is set, probability must be > 0'
            assert self.maintain_quorum, \
                'store_thrash = true must imply maintain_quorum = true'

        self.thread = gevent.spawn(self.do_thrash)

    def log(self, x):
        """
        locally log info messages
        """
        self.logger.info(x)

    def do_join(self):
        """
        Break out of this processes thrashing loop.
        """
        self.stopping = True
        self.thread.get()

    def should_thrash_store(self):
        """
        If allowed, indicate that we should thrash a certain percentage of
        the time as determined by the store_thrash_probability value.
        """
        if not self.store_thrash:
            return False
        return self.rng.randrange(0, 101) < self.store_thrash_probability

    def thrash_store(self, mon):
        """
        Thrash the monitor specified.
        :param mon: monitor to thrash
        """
        addr = self.ctx.ceph.conf['mon.%s' % mon]['mon addr']
        self.log('thrashing mon.{id}@{addr} store'.format(id=mon, addr=addr))
        out = self.manager.raw_cluster_cmd('-m', addr, 'sync', 'force')
        j = json.loads(out)
        assert j['ret'] == 0, \
            'error forcing store sync on mon.{id}:\n{ret}'.format(
                id=mon,ret=out)

    def should_freeze_mon(self):
        """
        Indicate that we should freeze a certain percentago of the time
        as determined by the freeze_mon_probability value.
        """
        return self.rng.randrange(0, 101) < self.freeze_mon_probability

    def freeze_mon(self, mon):
        """
        Send STOP signal to freeze the monitor.
        """
        log.info('Sending STOP to mon %s', mon)
        self.manager.signal_mon(mon, 19)  # STOP

    def unfreeze_mon(self, mon):
        """
        Send CONT signal to unfreeze the monitor.
        """
        log.info('Sending CONT to mon %s', mon)
        self.manager.signal_mon(mon, 18)  # CONT

    def kill_mon(self, mon):
        """
        Kill the monitor specified
        """
        self.log('killing mon.{id}'.format(id=mon))
        self.manager.kill_mon(mon)

    def revive_mon(self, mon):
        """
        Revive the monitor specified
        """
        self.log('killing mon.{id}'.format(id=mon))
        self.log('reviving mon.{id}'.format(id=mon))
        self.manager.revive_mon(mon)

    def max_killable(self):
        """
        Return the maximum number of monitors we can kill.
        """
        m = len(_get_mons(self.ctx))
        if self.maintain_quorum:
            return max(math.ceil(m/2.0)-1, 0)
        else:
            return m

    def do_thrash(self):
        """
        Cotinuously loop and thrash the monitors.
        """
        self.log('start thrashing')
        self.log('seed: {s}, revive delay: {r}, thrash delay: {t} '\
                   'thrash many: {tm}, maintain quorum: {mq} '\
                   'store thrash: {st}, probability: {stp} '\
                   'freeze mon: prob {fp} duration {fd}'.format(
                s=self.random_seed,r=self.revive_delay,t=self.thrash_delay,
                tm=self.thrash_many, mq=self.maintain_quorum,
                st=self.store_thrash,stp=self.store_thrash_probability,
                fp=self.freeze_mon_probability,fd=self.freeze_mon_duration,
                ))

        while not self.stopping:
            mons = _get_mons(self.ctx)
            self.manager.wait_for_mon_quorum_size(len(mons))
            self.log('making sure all monitors are in the quorum')
            for m in mons:
                s = self.manager.get_mon_status(m)
                assert s['state'] == 'leader' or s['state'] == 'peon'
                assert len(s['quorum']) == len(mons)

            kill_up_to = self.rng.randrange(1, self.max_killable()+1)
            mons_to_kill = self.rng.sample(mons, kill_up_to)
            self.log('monitors to thrash: {m}'.format(m=mons_to_kill))

            mons_to_freeze = []
            for mon in mons:
                if mon in mons_to_kill:
                    continue
                if self.should_freeze_mon():
                    mons_to_freeze.append(mon)
            self.log('monitors to freeze: {m}'.format(m=mons_to_freeze))

            for mon in mons_to_kill:
                self.log('thrashing mon.{m}'.format(m=mon))

                """ we only thrash stores if we are maintaining quorum """
                if self.should_thrash_store() and self.maintain_quorum:
                    self.thrash_store(mon)

                self.kill_mon(mon)

            if mons_to_freeze:
                for mon in mons_to_freeze:
                    self.freeze_mon(mon)
                self.log('waiting for {delay} secs to unfreeze mons'.format(
                    delay=self.freeze_mon_duration))
                time.sleep(self.freeze_mon_duration)
                for mon in mons_to_freeze:
                    self.unfreeze_mon(mon)

            if self.maintain_quorum:
                self.manager.wait_for_mon_quorum_size(len(mons)-len(mons_to_kill))
                for m in mons:
                    if m in mons_to_kill:
                        continue
                    s = self.manager.get_mon_status(m)
                    assert s['state'] == 'leader' or s['state'] == 'peon'
                    assert len(s['quorum']) == len(mons)-len(mons_to_kill)

            self.log('waiting for {delay} secs before reviving monitors'.format(
                delay=self.revive_delay))
            time.sleep(self.revive_delay)

            for mon in mons_to_kill:
                self.revive_mon(mon)
            # do more freezes
            if mons_to_freeze:
                for mon in mons_to_freeze:
                    self.freeze_mon(mon)
                self.log('waiting for {delay} secs to unfreeze mons'.format(
                    delay=self.freeze_mon_duration))
                time.sleep(self.freeze_mon_duration)
                for mon in mons_to_freeze:
                    self.unfreeze_mon(mon)

            self.manager.wait_for_mon_quorum_size(len(mons))
            for m in mons:
                s = self.manager.get_mon_status(m)
                assert s['state'] == 'leader' or s['state'] == 'peon'
                assert len(s['quorum']) == len(mons)

            if self.scrub:
                self.log('triggering scrub')
                try:
                    self.manager.raw_cluster_cmd('scrub')
                except Exception:
                    log.exception("Saw exception while triggering scrub")

            if self.thrash_delay > 0.0:
                self.log('waiting for {delay} secs before continuing thrashing'.format(
                    delay=self.thrash_delay))
                time.sleep(self.thrash_delay)

@contextlib.contextmanager
def task(ctx, config):
    """
    Stress test the monitor by thrashing them while another task/workunit
    is running.

    Please refer to MonitorThrasher class for further information on the
    available options.
    """
    if config is None:
        config = {}
    assert isinstance(config, dict), \
        'mon_thrash task only accepts a dict for configuration'
    assert len(_get_mons(ctx)) > 2, \
        'mon_thrash task requires at least 3 monitors'
    log.info('Beginning mon_thrash...')
    first_mon = teuthology.get_first_mon(ctx, config)
    (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
    manager = ceph_manager.CephManager(
        mon,
        ctx=ctx,
        logger=log.getChild('ceph_manager'),
        )
    thrash_proc = MonitorThrasher(ctx,
        manager, config,
        logger=log.getChild('mon_thrasher'))
    try:
        log.debug('Yielding')
        yield
    finally:
        log.info('joining mon_thrasher')
        thrash_proc.do_join()
        mons = _get_mons(ctx)
        manager.wait_for_mon_quorum_size(len(mons))
<|endoftext|>


<commit_before>from Timeseries import TimeSeries
import numpy as np

class ArrayTimeSeries(TimeSeries):
    """
    It is a new class ArrayTimeSeries which inherits from TimeSeries
    class and uses numpy arrays to store the data internally.

    Parameters
    ----------
    values : a sequence- data used to populate time series instance.
    times  : a sequence- time associated with each observation in `values`.

    >>>threes=range(0,1000,3)
    >>>times = range(0,1000,1)
    >>>ap=ArrayTimeSeries(times,threes)
    >>>ap[0] = 999
    >>>ap[0]
    999
    >>>ap
    ArrayTimeSeries(Length: 334, Times: array([  0,  ...97, 998, 999]), Values: array([999,  ...93, 996, 999]))
    >>>print(ap)
    ArrayTimeSeries with 334 elements (Times: array([  0,  ...97, 998, 999]), Values: array([999,  ...93, 996, 999]))
    >>>len(ap)
    334
    >>>ap = ap=ArrayTimeSeries([0,1,2,3],[5,6,7,8])
    >>>list(iter(ap))
    [5, 6, 7, 8]
    >>>list(ap.itertimes())
    [0, 1, 2, 3]
    >>>list(ap.iteritems())
    [(0, 5), (1, 6), (2, 7), (3, 8)]

    #Now test non-uniform time series
    >>>a = ArrayTimeSeries(times=[1,2.5,3,3.5,4],values=[0,5,10,8,7], )
    >>>a
    ArrayTimeSeries(Length: 5, Times: array([ 1. , ...,  3.5,  4. ]), Values: array([ 0,  5, 10,  8,  7]))
    >>>print(a)
    ArrayTimeSeries with 5 elements (Times: array([ 1. , ...,  3.5,  4. ]), Values: array([ 0,  5, 10,  8,  7]))

    """
    def __init__(self,times,values):
        TimeSeries.__init__(self,values,times)
        self._times = np.array(times)
        self._values = np.array(values)

    def __getitem__(self,index):
        try:
            return self._values[index]
        except IndexError:
            raise IndexError("Index out of bounds")

    def __setitem__(self,index,value):
        try:
            self._values[index] = value
        except IndexError:
            raise IndexError("Index out of bounds")

    def __len__(self):
        return super().__len__()

    def __iter__(self):
        return super().__iter__()

    def itertimes(self):
        return super().itertimes()

    def iteritems(self):
        return super().iteritems()

<commit_msg>#5 Fixed storage<commit_after>from Timeseries import TimeSeries
import numpy as np

class ArrayTimeSeries(TimeSeries):
    """
    It is a new class ArrayTimeSeries which inherits from TimeSeries
    class and uses numpy arrays to store the data internally.

    Parameters
    ----------
    values : a sequence- data used to populate time series instance.
    times  : a sequence- time associated with each observation in `values`.

    >>> threes=range(0,1000,3)
    >>> times = range(0,1000,1)
    >>> ap=ArrayTimeSeries(times,threes)
    >>> ap[0] = 999
    >>> ap[0]
    999
    >>> ap
    ArrayTimeSeries(Length: 334, Times: array([  0,  ...97, 998, 999]), Values: array([999,  ...93, 996, 999]))
    >>> print(ap)
    ArrayTimeSeries with 334 elements (Times: array([  0,  ...97, 998, 999]), Values: array([999,  ...93, 996, 999]))
    >>> len(ap)
    334
    >>> ap = ap=ArrayTimeSeries([0,1,2,3],[5,6,7,8])
    >>> list(iter(ap))
    [5, 6, 7, 8]
    >>> list(ap.itertimes())
    [0, 1, 2, 3]
    >>> list(ap.iteritems())
    [(0, 5), (1, 6), (2, 7), (3, 8)]

    #Now test non-uniform time series
    >>> a = ArrayTimeSeries(times=[1,2.5,3,3.5,4],values=[0,5,10,8,7], )
    >>> a
    ArrayTimeSeries(Length: 5, Times: array([ 1. , ...,  3.5,  4. ]), Values: array([ 0,  5, 10,  8,  7]))
    >>> print(a)
    ArrayTimeSeries with 5 elements (Times: array([ 1. , ...,  3.5,  4. ]), Values: array([ 0,  5, 10,  8,  7]))

    """
    def __init__(self,times,values):
        TimeSeries.__init__(self,values,times)
        self._times = np.array(times)
        self._values = np.array(values)

    def __getitem__(self,index):
        try:
            return self._values[index]
        except IndexError:
            raise IndexError("Index out of bounds")

    def __setitem__(self,index,value):
        try:
            self._values[index] = value
        except IndexError:
            raise IndexError("Index out of bounds")

    def __len__(self):
        return super().__len__()

    def __iter__(self):
        return super().__iter__()

    def itertimes(self):
        return super().itertimes()

    def iteritems(self):
        return super().iteritems()

<|endoftext|>


<commit_before>#!/usr/bin/python3

from flask import Flask

# Create an instance of Flask
app = Flask(__name__);

# Present a nice screen when someone is lost
@app.route("/")
def default_route():
    return "Welcome to the Coffee Build Server, where you may read error reports!";

# For servers, get their logs and put them in the database
@app.route("/logger/data/<string:arch>/<int:sid>")
def build_log_data(arch,sid):
    return "OMG! Someone messaged me!";

if __name__ == "__main__":
    app.run();
<commit_msg> - Add more features to buildserver<commit_after>#!/usr/bin/python3

from flask import Flask
from flask import request

# Create an instance of Flask
app = Flask(__name__);

# Present a nice screen when someone is lost
@app.route("/")
def default_route():
    return "Welcome to the Coffee Build Server, where you may read error reports!";

# For servers, get their logs and put them in the database
@app.route("/logger/data/<arch>/<int:sid>",methods=["POST","GET"])
def build_log_data(arch,sid):
    if request.method == "GET":
        return "Are you lost?";
    return "You are: plat=%s,server_id=%s" % (arch,sid);

if __name__ == "__main__":
    app.run();
<|endoftext|>


<commit_before># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

JAVA_DOCS="-link https://docs.oracle.com/javase/8/docs/api/"

def _impl(ctx):
  dir = ctx.label.name
  jar = ctx.outputs.jar

  dep_list = []
  for dep in ctx.files.deps:
    dep_list += [dep.path]

  src_list = []
  for src in ctx.files.srcs:
    src_list += [src.path]

  cmd = [
      "mkdir %s" % dir,
      "javadoc -quiet -tag onos.rsModel:a:\"onos model\" %s -d %s -cp %s %s" \
          % (JAVA_DOCS, dir, ":".join(dep_list), " ".join(src_list)),
      "jar cf %s -C %s ." % (jar.path, dir),
  ]

  ctx.action(
      inputs = ctx.files.srcs + ctx.files.deps,
      outputs = [jar],
      progress_message = "Generating javadocs jar for %s" %  ctx.attr.name,
      command = ";\n".join(cmd)
  )

javadoc = rule(
    attrs = {
        "deps": attr.label_list(allow_files = True),
        "srcs": attr.label_list(allow_files = True),
    },
    implementation = _impl,
    outputs = {"jar" : "%{name}.jar"},
)

<commit_msg>Use UTF8 encoding for javadocs<commit_after># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

JAVA_DOCS="-link https://docs.oracle.com/javase/8/docs/api/"

def _impl(ctx):
  dir = ctx.label.name
  jar = ctx.outputs.jar

  dep_list = []
  for dep in ctx.files.deps:
    dep_list += [dep.path]

  src_list = []
  for src in ctx.files.srcs:
    src_list += [src.path]

  cmd = [
      "mkdir %s" % dir,
      "javadoc -encoding utf8 -quiet -tag onos.rsModel:a:\"onos model\" %s -d %s -cp %s %s" \
          % (JAVA_DOCS, dir, ":".join(dep_list), " ".join(src_list)),
      "jar cf %s -C %s ." % (jar.path, dir),
  ]

  ctx.action(
      inputs = ctx.files.srcs + ctx.files.deps,
      outputs = [jar],
      progress_message = "Generating javadocs jar for %s" %  ctx.attr.name,
      command = ";\n".join(cmd)
  )

javadoc = rule(
    attrs = {
        "deps": attr.label_list(allow_files = True),
        "srcs": attr.label_list(allow_files = True),
    },
    implementation = _impl,
    outputs = {"jar" : "%{name}.jar"},
)

<|endoftext|>


<commit_before>import GPy
import numpy as np
import matplotlib.pyplot as plt
from GPy.util import datasets

def student_t_approx(optimize=True, plot=True):
    """
    Example of regressing with a student t likelihood using Laplace
    """
    real_std = 0.1
    #Start a function, any function
    X = np.linspace(0.0, np.pi*2, 100)[:, None]
    Y = np.sin(X) + np.random.randn(*X.shape)*real_std
    Y = Y/Y.max()
    Yc = Y.copy()

    X_full = np.linspace(0.0, np.pi*2, 500)[:, None]
    Y_full = np.sin(X_full)
    Y_full = Y_full/Y_full.max()

    #Slightly noisy data
    Yc[75:80] += 1

    #Very noisy data
    #Yc[10] += 100
    #Yc[25] += 10
    #Yc[23] += 10
    #Yc[26] += 1000
    #Yc[24] += 10
    #Yc = Yc/Yc.max()

    #Add student t random noise to datapoints
    deg_free = 5
    print "Real noise: ", real_std
    initial_var_guess = 0.5
    edited_real_sd = initial_var_guess

    # Kernel object
    kernel1 = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1])
    kernel2 = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1])
    kernel3 = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1])
    kernel4 = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1])

    #Gaussian GP model on clean data
    #m1 = GPy.models.GPRegression(X, Y.copy(), kernel=kernel1)
    ## optimize
    #m1['white'].constrain_fixed(1e-5)
    #m1.randomize()

    ##Gaussian GP model on corrupt data
    #m2 = GPy.models.GPRegression(X, Yc.copy(), kernel=kernel2)
    #m1['white'].constrain_fixed(1e-5)
    #m2.randomize()

    #Student t GP model on clean data
    t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)
    laplace_inf = GPy.inference.latent_function_inference.Laplace()
    m3 = GPy.core.GP(X, Y.copy(), kernel3, likelihood=t_distribution, inference_method=laplace_inf)
    m3['t_noise'].constrain_bounded(1e-6, 10.)
    m3['white'].constrain_fixed(1e-5)
    m3.randomize()
    debug = True
    print m3
    if debug:
        m3.optimize(messages=1)
        return m3

    #Student t GP model on corrupt data
    t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)
    laplace_inf = GPy.inference.latent_function_inference.Laplace()
    m4 = GPy.core.GP(X, Yc.copy(), kernel4, likelihood=t_distribution, inference_method=laplace_inf)
    m4['t_noise'].constrain_bounded(1e-6, 10.)
    m4['white'].constrain_fixed(1e-5)
    m4.randomize()

    if optimize:
        optimizer='scg'
        print "Clean Gaussian"
        m1.optimize(optimizer, messages=1)
        print "Corrupt Gaussian"
        m2.optimize(optimizer, messages=1)
        print "Clean student t"
        m3.optimize(optimizer, messages=1)
        print "Corrupt student t"
        m4.optimize(optimizer, messages=1)

    if plot:
        plt.figure(1)
        plt.suptitle('Gaussian likelihood')
        ax = plt.subplot(211)
        m1.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Gaussian clean')

        ax = plt.subplot(212)
        m2.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Gaussian corrupt')

        plt.figure(2)
        plt.suptitle('Student-t likelihood')
        ax = plt.subplot(211)
        m3.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Student-t rasm clean')

        ax = plt.subplot(212)
        m4.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Student-t rasm corrupt')

    return m1, m2, m3, m4

def boston_example(optimize=True, plot=True):
    import sklearn
    from sklearn.cross_validation import KFold
    optimizer='bfgs'
    messages=0
    data = datasets.boston_housing()
    degrees_freedoms = [3, 5, 8, 10]
    X = data['X'].copy()
    Y = data['Y'].copy()
    X = X-X.mean(axis=0)
    X = X/X.std(axis=0)
    Y = Y-Y.mean()
    Y = Y/Y.std()
    num_folds = 10
    kf = KFold(len(Y), n_folds=num_folds, indices=True)
    num_models = len(degrees_freedoms) + 3 #3 for baseline, gaussian, gaussian laplace approx
    score_folds = np.zeros((num_models, num_folds))
    pred_density = score_folds.copy()

    def rmse(Y, Ystar):
        return np.sqrt(np.mean((Y-Ystar)**2))

    for n, (train, test) in enumerate(kf):
        X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        print "Fold {}".format(n)

        noise = 1e-1 #np.exp(-2)
        rbf_len = 0.5
        data_axis_plot = 4
        kernelstu = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])
        kernelgp = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])

        #Baseline
        score_folds[0, n] = rmse(Y_test, np.mean(Y_train))

        #Gaussian GP
        print "Gauss GP"
        mgp = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelgp.copy())
        mgp.constrain_fixed('white', 1e-5)
        mgp['rbf_len'] = rbf_len
        mgp['noise'] = noise
        print mgp
        if optimize:
            mgp.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mgp.predict(X_test)
        score_folds[1, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[1, n] = np.mean(mgp.log_predictive_density(X_test, Y_test))
        print mgp
        print pred_density

        print "Gaussian Laplace GP"
        N, D = Y_train.shape
        g_distribution = GPy.likelihoods.noise_model_constructors.gaussian(variance=noise, N=N, D=D)
        g_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), g_distribution)
        mg = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=g_likelihood)
        mg.constrain_positive('noise_variance')
        mg.constrain_fixed('white', 1e-5)
        mg['rbf_len'] = rbf_len
        mg['noise'] = noise
        print mg
        if optimize:
            mg.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mg.predict(X_test)
        score_folds[2, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[2, n] = np.mean(mg.log_predictive_density(X_test, Y_test))
        print pred_density
        print mg

        for stu_num, df in enumerate(degrees_freedoms):
            #Student T
            print "Student-T GP {}df".format(df)
            t_distribution = GPy.likelihoods.noise_model_constructors.student_t(deg_free=df, sigma2=noise)
            stu_t_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), t_distribution)
            mstu_t = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=stu_t_likelihood)
            mstu_t.constrain_fixed('white', 1e-5)
            mstu_t.constrain_bounded('t_noise', 0.0001, 1000)
            mstu_t['rbf_len'] = rbf_len
            mstu_t['t_noise'] = noise
            print mstu_t
            if optimize:
                mstu_t.optimize(optimizer=optimizer, messages=messages)
            Y_test_pred = mstu_t.predict(X_test)
            score_folds[3+stu_num, n] = rmse(Y_test, Y_test_pred[0])
            pred_density[3+stu_num, n] = np.mean(mstu_t.log_predictive_density(X_test, Y_test))
            print pred_density
            print mstu_t

    if plot:
        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('GP gauss')

        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('Lap gauss')

        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('Stu t {}df'.format(df))

    print "Average scores: {}".format(np.mean(score_folds, 1))
    print "Average pred density: {}".format(np.mean(pred_density, 1))

    if plot:
        #Plotting
        stu_t_legends = ['Student T, df={}'.format(df) for df in degrees_freedoms]
        legends = ['Baseline', 'Gaussian', 'Laplace Approx Gaussian'] + stu_t_legends

        #Plot boxplots for RMSE density
        fig = plt.figure()
        ax=fig.add_subplot(111)
        plt.title('RMSE')
        bp = ax.boxplot(score_folds.T, notch=0, sym='+', vert=1, whis=1.5)
        plt.setp(bp['boxes'], color='black')
        plt.setp(bp['whiskers'], color='black')
        plt.setp(bp['fliers'], color='red', marker='+')
        xtickNames = plt.setp(ax, xticklabels=legends)
        plt.setp(xtickNames, rotation=45, fontsize=8)
        ax.set_ylabel('RMSE')
        ax.set_xlabel('Distribution')
        #Make grid and put it below boxes
        ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                alpha=0.5)
        ax.set_axisbelow(True)

        #Plot boxplots for predictive density
        fig = plt.figure()
        ax=fig.add_subplot(111)
        plt.title('Predictive density')
        bp = ax.boxplot(pred_density[1:,:].T, notch=0, sym='+', vert=1, whis=1.5)
        plt.setp(bp['boxes'], color='black')
        plt.setp(bp['whiskers'], color='black')
        plt.setp(bp['fliers'], color='red', marker='+')
        xtickNames = plt.setp(ax, xticklabels=legends[1:])
        plt.setp(xtickNames, rotation=45, fontsize=8)
        ax.set_ylabel('Mean Log probability P(Y*|Y)')
        ax.set_xlabel('Distribution')
        #Make grid and put it below boxes
        ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                alpha=0.5)
        ax.set_axisbelow(True)
    return mstu_t

#def precipitation_example():
    #import sklearn
    #from sklearn.cross_validation import KFold
    #data = datasets.boston_housing()
    #X = data['X'].copy()
    #Y = data['Y'].copy()
    #X = X-X.mean(axis=0)
    #X = X/X.std(axis=0)
    #Y = Y-Y.mean()
    #Y = Y/Y.std()
    #import ipdb; ipdb.set_trace()  # XXX BREAKPOINT
    #num_folds = 10
    #kf = KFold(len(Y), n_folds=num_folds, indices=True)
    #score_folds = np.zeros((4, num_folds))
    #def rmse(Y, Ystar):
        #return np.sqrt(np.mean((Y-Ystar)**2))
    ##for train, test in kf:
    #for n, (train, test) in enumerate(kf):
        #X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        #print "Fold {}".format(n)

<commit_msg>Fixed non_gaussian demo<commit_after>import GPy
import numpy as np
import matplotlib.pyplot as plt
from GPy.util import datasets

def student_t_approx(optimize=True, plot=True):
    """
    Example of regressing with a student t likelihood using Laplace
    """
    real_std = 0.1
    #Start a function, any function
    X = np.linspace(0.0, np.pi*2, 100)[:, None]
    Y = np.sin(X) + np.random.randn(*X.shape)*real_std
    Y = Y/Y.max()
    Yc = Y.copy()

    X_full = np.linspace(0.0, np.pi*2, 500)[:, None]
    Y_full = np.sin(X_full)
    Y_full = Y_full/Y_full.max()

    #Slightly noisy data
    Yc[75:80] += 1

    #Very noisy data
    #Yc[10] += 100
    #Yc[25] += 10
    #Yc[23] += 10
    #Yc[26] += 1000
    #Yc[24] += 10
    #Yc = Yc/Yc.max()

    #Add student t random noise to datapoints
    deg_free = 5
    print "Real noise: ", real_std
    initial_var_guess = 0.5
    edited_real_sd = initial_var_guess

    # Kernel object
    kernel1 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel2 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel3 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel4 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])

    #Gaussian GP model on clean data
    m1 = GPy.models.GPRegression(X, Y.copy(), kernel=kernel1)
    # optimize
    m1['white'].constrain_fixed(1e-5)
    m1.randomize()

    #Gaussian GP model on corrupt data
    m2 = GPy.models.GPRegression(X, Yc.copy(), kernel=kernel2)
    m2['white'].constrain_fixed(1e-5)
    m2.randomize()

    #Student t GP model on clean data
    t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)
    laplace_inf = GPy.inference.latent_function_inference.Laplace()
    m3 = GPy.core.GP(X, Y.copy(), kernel3, likelihood=t_distribution, inference_method=laplace_inf)
    m3['t_noise'].constrain_bounded(1e-6, 10.)
    m3['white'].constrain_fixed(1e-5)
    m3.randomize()
    debug = True
    print m3
    if debug:
        m3.optimize(messages=1)
        return m3

    #Student t GP model on corrupt data
    t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)
    laplace_inf = GPy.inference.latent_function_inference.Laplace()
    m4 = GPy.core.GP(X, Yc.copy(), kernel4, likelihood=t_distribution, inference_method=laplace_inf)
    m4['t_noise'].constrain_bounded(1e-6, 10.)
    m4['white'].constrain_fixed(1e-5)
    m4.randomize()

    if optimize:
        optimizer='scg'
        print "Clean Gaussian"
        m1.optimize(optimizer, messages=1)
        print "Corrupt Gaussian"
        m2.optimize(optimizer, messages=1)
        print "Clean student t"
        m3.optimize(optimizer, messages=1)
        print "Corrupt student t"
        m4.optimize(optimizer, messages=1)

    if plot:
        plt.figure(1)
        plt.suptitle('Gaussian likelihood')
        ax = plt.subplot(211)
        m1.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Gaussian clean')

        ax = plt.subplot(212)
        m2.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Gaussian corrupt')

        plt.figure(2)
        plt.suptitle('Student-t likelihood')
        ax = plt.subplot(211)
        m3.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Student-t rasm clean')

        ax = plt.subplot(212)
        m4.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Student-t rasm corrupt')

    return m1, m2, m3, m4

def boston_example(optimize=True, plot=True):
    import sklearn
    from sklearn.cross_validation import KFold
    optimizer='bfgs'
    messages=0
    data = datasets.boston_housing()
    degrees_freedoms = [3, 5, 8, 10]
    X = data['X'].copy()
    Y = data['Y'].copy()
    X = X-X.mean(axis=0)
    X = X/X.std(axis=0)
    Y = Y-Y.mean()
    Y = Y/Y.std()
    num_folds = 10
    kf = KFold(len(Y), n_folds=num_folds, indices=True)
    num_models = len(degrees_freedoms) + 3 #3 for baseline, gaussian, gaussian laplace approx
    score_folds = np.zeros((num_models, num_folds))
    pred_density = score_folds.copy()

    def rmse(Y, Ystar):
        return np.sqrt(np.mean((Y-Ystar)**2))

    for n, (train, test) in enumerate(kf):
        X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        print "Fold {}".format(n)

        noise = 1e-1 #np.exp(-2)
        rbf_len = 0.5
        data_axis_plot = 4
        kernelstu = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])
        kernelgp = GPy.kern.rbf(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])

        #Baseline
        score_folds[0, n] = rmse(Y_test, np.mean(Y_train))

        #Gaussian GP
        print "Gauss GP"
        mgp = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelgp.copy())
        mgp.constrain_fixed('white', 1e-5)
        mgp['rbf_len'] = rbf_len
        mgp['noise'] = noise
        print mgp
        if optimize:
            mgp.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mgp.predict(X_test)
        score_folds[1, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[1, n] = np.mean(mgp.log_predictive_density(X_test, Y_test))
        print mgp
        print pred_density

        print "Gaussian Laplace GP"
        N, D = Y_train.shape
        g_distribution = GPy.likelihoods.noise_model_constructors.gaussian(variance=noise, N=N, D=D)
        g_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), g_distribution)
        mg = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=g_likelihood)
        mg.constrain_positive('noise_variance')
        mg.constrain_fixed('white', 1e-5)
        mg['rbf_len'] = rbf_len
        mg['noise'] = noise
        print mg
        if optimize:
            mg.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mg.predict(X_test)
        score_folds[2, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[2, n] = np.mean(mg.log_predictive_density(X_test, Y_test))
        print pred_density
        print mg

        for stu_num, df in enumerate(degrees_freedoms):
            #Student T
            print "Student-T GP {}df".format(df)
            t_distribution = GPy.likelihoods.noise_model_constructors.student_t(deg_free=df, sigma2=noise)
            stu_t_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), t_distribution)
            mstu_t = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=stu_t_likelihood)
            mstu_t.constrain_fixed('white', 1e-5)
            mstu_t.constrain_bounded('t_noise', 0.0001, 1000)
            mstu_t['rbf_len'] = rbf_len
            mstu_t['t_noise'] = noise
            print mstu_t
            if optimize:
                mstu_t.optimize(optimizer=optimizer, messages=messages)
            Y_test_pred = mstu_t.predict(X_test)
            score_folds[3+stu_num, n] = rmse(Y_test, Y_test_pred[0])
            pred_density[3+stu_num, n] = np.mean(mstu_t.log_predictive_density(X_test, Y_test))
            print pred_density
            print mstu_t

    if plot:
        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('GP gauss')

        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('Lap gauss')

        plt.figure()
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('Stu t {}df'.format(df))

    print "Average scores: {}".format(np.mean(score_folds, 1))
    print "Average pred density: {}".format(np.mean(pred_density, 1))

    if plot:
        #Plotting
        stu_t_legends = ['Student T, df={}'.format(df) for df in degrees_freedoms]
        legends = ['Baseline', 'Gaussian', 'Laplace Approx Gaussian'] + stu_t_legends

        #Plot boxplots for RMSE density
        fig = plt.figure()
        ax=fig.add_subplot(111)
        plt.title('RMSE')
        bp = ax.boxplot(score_folds.T, notch=0, sym='+', vert=1, whis=1.5)
        plt.setp(bp['boxes'], color='black')
        plt.setp(bp['whiskers'], color='black')
        plt.setp(bp['fliers'], color='red', marker='+')
        xtickNames = plt.setp(ax, xticklabels=legends)
        plt.setp(xtickNames, rotation=45, fontsize=8)
        ax.set_ylabel('RMSE')
        ax.set_xlabel('Distribution')
        #Make grid and put it below boxes
        ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                alpha=0.5)
        ax.set_axisbelow(True)

        #Plot boxplots for predictive density
        fig = plt.figure()
        ax=fig.add_subplot(111)
        plt.title('Predictive density')
        bp = ax.boxplot(pred_density[1:,:].T, notch=0, sym='+', vert=1, whis=1.5)
        plt.setp(bp['boxes'], color='black')
        plt.setp(bp['whiskers'], color='black')
        plt.setp(bp['fliers'], color='red', marker='+')
        xtickNames = plt.setp(ax, xticklabels=legends[1:])
        plt.setp(xtickNames, rotation=45, fontsize=8)
        ax.set_ylabel('Mean Log probability P(Y*|Y)')
        ax.set_xlabel('Distribution')
        #Make grid and put it below boxes
        ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                alpha=0.5)
        ax.set_axisbelow(True)
    return mstu_t

#def precipitation_example():
    #import sklearn
    #from sklearn.cross_validation import KFold
    #data = datasets.boston_housing()
    #X = data['X'].copy()
    #Y = data['Y'].copy()
    #X = X-X.mean(axis=0)
    #X = X/X.std(axis=0)
    #Y = Y-Y.mean()
    #Y = Y/Y.std()
    #import ipdb; ipdb.set_trace()  # XXX BREAKPOINT
    #num_folds = 10
    #kf = KFold(len(Y), n_folds=num_folds, indices=True)
    #score_folds = np.zeros((4, num_folds))
    #def rmse(Y, Ystar):
        #return np.sqrt(np.mean((Y-Ystar)**2))
    ##for train, test in kf:
    #for n, (train, test) in enumerate(kf):
        #X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        #print "Fold {}".format(n)

<|endoftext|>


<commit_before>#!/usr/bin/python
import distutils.spawn
import json
import os
import re
import sys
import xml.dom.minidom
from xml.etree.ElementTree import ElementTree,SubElement,Element,dump,tostring

"""
Check if deps are present (ImageMagick)
"""
def checkProgram(program):
    return distutils.spawn.find_executable(program)

"""
DICOM image metadata indentation-based parser class
"""
class ImageMagickIdentifyParser:

    Data = None
    HISTOGRAM_ELEM="HistogramLevel"
    # RE_GROUPED_ENTRY examples:
    # dcm:DeviceSerialNumber
    RE_GROUPED_ENTRY = r"(?P<prefix>.+?):(?P<name>.*)$"
    # RE_LINE_GENERIC examples:
    # Page geometry: 512x512+0+0
    RE_LINE_GENERIC = r"^(?P<leading>\s*)(?P<name>.*):(?P<value>\s.*|)$"
    # RE_LINE_HISTO examples:
    # 30489: (  385,  385,  385) #018101810181 gray(0.587472%,0.587472%,0.587472%)
    # 6709: (    0,    0,    0) #000000000000 gray(0,0,0)
    # 6709: (    0,    0,    0) #000000000000 gray(0)
    # 16680: (  128,  128,  128) #008000800080 gray(0.195315%)
    # 25206: (  256,  256,  256) #010001000100 gray(0.390631%)
    #
    # Note that the last list of numbers in the parenthesis can have either 3 or 1 value.
    RE_LINE_HISTO = r"^\s+(?P<count>\d+):\s*\(\s*(?P<rval>\d+)\s*,\s*(?P<gval>\d+)\s*,\s*(?P<bval>\d+)\s*\)\s*#(?P<hexval>[0-9A-F]{12})\s*(?P<colname>[a-zA-Z]+)\s*\((?:(?P<rperc>\d+(?:\.\d+)?)%?,(?P<gperc>\d+(?:\.\d+)?)%?,(?P<bperc>\d+(?:\.\d+)?)%?|(?P<gray>\d+(?:\.\d+)?)%?)\)"
    def __init__(self):
        if not checkProgram('identify'):
            raise Exception('[Error] ImageMagick is missing')

        # reset internal data
        self.Data = {}

    # adapt the tag name to conform with the XML format
    def normalizeName(self, name):
        name = re.sub(r'\s','_',name)
        name = re.sub(r'[^a-zA-Z0-9]','_',name)
        # trim any underscores at the end
        name = re.sub(r'_+$','',name)
        # and at the beginning
        name = re.sub(r'^_+','',name)

        def upperCallback(x):
            return x.group(1).upper()

        # apply regex to convert to camelCase
        # assuming it's in underscore format
        ccName = re.sub(r'_(.)',upperCallback,name)
        return ccName

    def runCmd(self, cmd):
        return os.popen(cmd).read()

    def parseLineGeneric(self, line):
        matchGeneric = re.match(self.RE_LINE_GENERIC, line, re.UNICODE)
        if not matchGeneric:
            return None

        # extract the values picked up by the regex above
        d = matchGeneric.groupdict()
        name = d['name']
        value = d['value']
        leading = d['leading']

        # clean up leading and trailing whitespace
        value = re.sub(r'^\s+','',value)
        value = re.sub(r'\s+$','',value)

        # get the current level using the indentation
        lc = len(leading)/2
        # everything is shifted one level deeper, because we have a root node
        lc += 1

        # a new node is created to store the information extracted
        new_node = {
                'name': name, \
                'value': value, \
                'children': [], \
                'level': lc, \
                'parent': None,\
                }
        return new_node

    def parseLineHisto(self, line, level):
        matchHisto = re.match(self.RE_LINE_HISTO, line, re.UNICODE)
        if not matchHisto:
            return None
        d = matchHisto.groupdict()
        new_node = d
        new_node['name'] = self.HISTOGRAM_ELEM
        new_node['value'] = ''
        new_node['children'] = []
        new_node['level'] = level
        new_node['parent'] = None
        return new_node

    def parse(self, filePath):
        self.parseRaw(filePath)
        self.treeTransformGroup()

    def parseRaw(self, filePath):
        # get identify output
        output = self.runCmd('identify -verbose ' + filePath)
        output = output.decode('iso-8859-1').encode('utf-8')
        lines = output.split('\n')

        #### First pass: building the AST
        # initialize the stack with a root node
        stack = 200 * [None]
        root = {'children': [], 'parent': None, 'name': '', 'value': ''}
        stack[0] = root

        # flag that indicates the histogram parsing mode is on
        hm = False
        # initialize current level, previous level, histogram parsing level
        lc,lp,lh = 1,0,0

        for line in lines:
            newNode = None

            if hm:
                newNode = self.parseLineHisto(line, lh)
                # we failed parsing the histogram line, assume we're back to generic lines
                # the current line needs to be reparsed as a generic line
                if not newNode:
                    hm = False
                    newNode = self.parseLineGeneric(line)
            else:
                newNode = self.parseLineGeneric(line)
                if newNode and 'name' in newNode and newNode['name'] == 'Histogram':
                    # if we encounter the histogram node, we turn on histogram parsing mode
                    hm = True
                    # and we store the level for all the upcoming histogram lines that follow
                    lh = 1 + newNode['level']

            if newNode:
                lc = newNode['level']

                # dispose of the 'level' attribute, we only need that information here
                if 'level' in newNode:
                    lc = newNode['level']
                    del newNode['level']

                # set parent
                newNode['parent'] = stack[lc-1]
                # add the node as a child of its immediate parent
                (stack[lc-1])['children'].append(newNode)
                # put the node on the stack, this will subsequently be used
                # by the the next iterations of this loop(if this node has children)
                stack[lc] = newNode
                # update the previous level
                lp = lc

        # store the tree in the class attribute for later use
        self.Data = root

    ## Group multiple options with the same
    ## prefix before the colon into a new parent
    ## having the common prefix as the name
    ##
    ## an example to illustrate this:
    ## <x>
    ##

    ##

    ## </x>
    ##
    ## =>
    ##
    ## <x>
    ##

    ##
    ##
    ##

    ## </x>
    def treeTransformGroup(self):
        root = self.Data
        stack = []
        stack.append(root)

        while len(stack) > 0:
            # We're visiting the next immediate node on the stack
            # (regular DFS traversal)
            x = stack.pop()
            if 'children' in x:
                y = x['children']
                i = 0
                ## a's keys will be common prefixes
                ## and the values will be new parent nodes which hold the
                ## children that will be transfered from x
                a = {}
                while i < len(y):
                    z = y[i]
                    match = re.match(self.RE_GROUPED_ENTRY, z.get('name',''), re.UNICODE)
                    if match:
                        d = match.groupdict()
                        dPrefix = d['prefix']
                        dName = d['name']
                        if dName == '':
                            del y[i]
                            continue
                        p = a.setdefault(dPrefix, \
                                {
                                    'children': [],
                                    'name': dPrefix,
                                    'value': '',
                                    'parent': x,
                                })
                        # update z's parent because it has been moved.
                        # to illustrate this, here is how the hierarchy changes:
                        # x->z => x->p->z
                        # so p is the new parent of x
                        z['parent'] = p
                        z['name'] = dName
                        # add z to p's children
                        p['children'] += [z]
                        del y[i]
                    else:
                        i += 1
                # at this point, depending on whether there were children to be grouped
                # they were(into a), and those that couldn't be will have stayed the same(in x['children']).
                # at this point, some nodes (some children of x) have been displaced and are now
                # children of new parent nodes, we are now expressing the fact that x has, among other children,
                # these new parent children that were created
                x['children'] += a.values()

            # Get x's children and put them on the stack
            # (continue the regular DFS)
            if 'children' in x:
                stack += x['children']

    # Note: this is only to be used for JSON serialization
    def treeTransformCompact(self, x):
        # this is basically a rebuilding of the tree in a more compact form.
        # here is a summary of what this method does:
        #
        # 1) we transform the tree as follows:
        # we aim to replace the 'children' attribute with either
        # an array or a dictionary, as follows:
        #
        # - all children have different names => we can store them in a dict
        # - if at least two children have the same name => we need to store them in an array
        #
        # 2) if a node has no children, and it has no additional attributes, then it
        # can be expressed as {k: v}
        #
        # check if it has no children
        xHasNoChildren = ('children' not in x) or ('children' in x and len(x['children']) == 0)
        # check if it has only basic properties: name,value,parent,children
        xHasOnlyNameValue = ('name' in x and 'value' in x and len(x.keys()) <= 4)

        # strip tree of parent attributes in order to avoid
        # circular references when serializing
        # to json
        del x['parent']

        if xHasNoChildren:
            del x['children']

            # x has no children
            if xHasOnlyNameValue:
                k = x['name']
                v = x['value']
                return [1,k,v]
            else:
                xname = x['name']
                xvalue = x['value']
                del x['name']
                del x['value']
                x['_value'] = xvalue
                return [2,xname,x]
        else:
            c = []
            xname = x['name']
            xvalue = x['value']
            # has children, recurse into children
            i = 0
            while i < len(x['children']):
                yi = x['children'][i]
                zi = self.treeTransformCompact(yi)
                c.append(zi)
                i += 1

            # constructing the new node
            w = None
            cnames = map(lambda z: z[1], c)
            if len(set(cnames)) == len(cnames):
                # the children all have distinct names, so w will be a dict
                w = {}
                for z in c:
                    w[z[1]] = z[2]
            else:
                # name collision are present, we need an array
                w = []
                for z in c:
                    w.append({z[1]: z[2]})

            return [3,xname,w]

    def serializeXML(self,root,xmlRoot):
        name = root['name']
        value = root['value']
        name = self.normalizeName(name)
        name = name.decode('utf-8')
        value = value.decode('utf-8')

        # serialize the node
        if 'children' in root and len(root['children']) > 0:
            for c in root['children']:
                cName = self.normalizeName(c['name'])
                xmlChild = SubElement(xmlRoot,cName)
                self.serializeXML(c,xmlChild)
        else:
            if name == self.HISTOGRAM_ELEM:
                xmlRoot.set('n', root['count'])
                xmlRoot.tag = self.HISTOGRAM_ELEM
                for k,v in root.iteritems():
                    # guard against undefined values(these are coming from the captures
                    # in the RE_LINE_HISTO regex, and the xml module will throw exceptions on
                    # the undefined values, so we want to avoid that)
                    # and check that the key is not an internal data key
                    if v and k not in ['name','value','parent','children']:
                        xmlRoot.set(k,v)
            else:
                xmlRoot.text=value

    def toJSON(self):
        Data = self.Data.copy()
        # run transformation to compact tree
        Data = self.treeTransformCompact(Data)
        # serialize to json
        return json.dumps(Data[2], indent=2)

    def toXML(self):
        Data = self.Data.copy()
        root = Data['children'][0]
        # serialize the root node and return it
        tree = ElementTree(Element('Image'))
        tree.getroot().set('file',Data['children'][0]['value'])
        self.serializeXML(root, tree.getroot())
        #x = xml.dom.minidom.parseString(tostring(tree.getroot(),encoding='utf8'))
        return tostring(tree.getroot(),encoding='utf8')

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='DICOM image metadata convertor')
    parser.add_argument("-i", dest="filename", required=True, help="input file")
    parser.add_argument('--raw' , '-r', dest='raw', action='store_true')
    parser.add_argument('--json', '-j', dest='json', action='store_true')
    parser.add_argument('--xml' , '-x', dest='xml', action='store_true')
    args = parser.parse_args()
    o = ImageMagickIdentifyParser()
    o.parse(args.filename)
    if args.raw:
        print o.Data
    if args.json:
        print o.toJSON()
    if args.xml:
        print o.toXML()
<commit_msg>added pretty-printing for XML<commit_after>#!/usr/bin/python
import distutils.spawn
import json
import os
import re
import sys
from xml.dom import minidom
from xml.etree.ElementTree import ElementTree,SubElement,Element,dump,tostring

"""
Check if deps are present (ImageMagick)
"""
def checkProgram(program):
    return distutils.spawn.find_executable(program)

"""
DICOM image metadata indentation-based parser class
"""
class ImageMagickIdentifyParser:

    Data = None
    HISTOGRAM_ELEM="HistogramLevel"
    # RE_GROUPED_ENTRY examples:
    # dcm:DeviceSerialNumber
    RE_GROUPED_ENTRY = r"(?P<prefix>.+?):(?P<name>.*)$"
    # RE_LINE_GENERIC examples:
    # Page geometry: 512x512+0+0
    RE_LINE_GENERIC = r"^(?P<leading>\s*)(?P<name>.*):(?P<value>\s.*|)$"
    # RE_LINE_HISTO examples:
    # 30489: (  385,  385,  385) #018101810181 gray(0.587472%,0.587472%,0.587472%)
    # 6709: (    0,    0,    0) #000000000000 gray(0,0,0)
    # 6709: (    0,    0,    0) #000000000000 gray(0)
    # 16680: (  128,  128,  128) #008000800080 gray(0.195315%)
    # 25206: (  256,  256,  256) #010001000100 gray(0.390631%)
    #
    # Note that the last list of numbers in the parenthesis can have either 3 or 1 value.
    RE_LINE_HISTO = r"^\s+(?P<count>\d+):\s*\(\s*(?P<rval>\d+)\s*,\s*(?P<gval>\d+)\s*,\s*(?P<bval>\d+)\s*\)\s*#(?P<hexval>[0-9A-F]{12})\s*(?P<colname>[a-zA-Z]+)\s*\((?:(?P<rperc>\d+(?:\.\d+)?)%?,(?P<gperc>\d+(?:\.\d+)?)%?,(?P<bperc>\d+(?:\.\d+)?)%?|(?P<gray>\d+(?:\.\d+)?)%?)\)"
    def __init__(self):
        if not checkProgram('identify'):
            raise Exception('[Error] ImageMagick is missing')

        # reset internal data
        self.Data = {}

    # adapt the tag name to conform with the XML format
    def normalizeName(self, name):
        name = re.sub(r'\s','_',name)
        name = re.sub(r'[^a-zA-Z0-9]','_',name)
        # trim any underscores at the end
        name = re.sub(r'_+$','',name)
        # and at the beginning
        name = re.sub(r'^_+','',name)

        def upperCallback(x):
            return x.group(1).upper()

        # apply regex to convert to camelCase
        # assuming it's in underscore format
        ccName = re.sub(r'_(.)',upperCallback,name)
        return ccName

    def runCmd(self, cmd):
        return os.popen(cmd).read()

    def parseLineGeneric(self, line):
        matchGeneric = re.match(self.RE_LINE_GENERIC, line, re.UNICODE)
        if not matchGeneric:
            return None

        # extract the values picked up by the regex above
        d = matchGeneric.groupdict()
        name = d['name']
        value = d['value']
        leading = d['leading']

        # clean up leading and trailing whitespace
        value = re.sub(r'^\s+','',value)
        value = re.sub(r'\s+$','',value)

        # get the current level using the indentation
        lc = len(leading)/2
        # everything is shifted one level deeper, because we have a root node
        lc += 1

        # a new node is created to store the information extracted
        new_node = {
                'name': name, \
                'value': value, \
                'children': [], \
                'level': lc, \
                'parent': None,\
                }
        return new_node

    def parseLineHisto(self, line, level):
        matchHisto = re.match(self.RE_LINE_HISTO, line, re.UNICODE)
        if not matchHisto:
            return None
        d = matchHisto.groupdict()
        new_node = d
        new_node['name'] = self.HISTOGRAM_ELEM
        new_node['value'] = ''
        new_node['children'] = []
        new_node['level'] = level
        new_node['parent'] = None
        return new_node

    def parse(self, filePath):
        self.parseRaw(filePath)
        self.treeTransformGroup()

    def parseRaw(self, filePath):
        # get identify output
        output = self.runCmd('identify -verbose ' + filePath)
        output = output.decode('iso-8859-1').encode('utf-8')
        lines = output.split('\n')

        #### First pass: building the AST
        # initialize the stack with a root node
        stack = 200 * [None]
        root = {'children': [], 'parent': None, 'name': '', 'value': ''}
        stack[0] = root

        # flag that indicates the histogram parsing mode is on
        hm = False
        # initialize current level, previous level, histogram parsing level
        lc,lp,lh = 1,0,0

        for line in lines:
            newNode = None

            if hm:
                newNode = self.parseLineHisto(line, lh)
                # we failed parsing the histogram line, assume we're back to generic lines
                # the current line needs to be reparsed as a generic line
                if not newNode:
                    hm = False
                    newNode = self.parseLineGeneric(line)
            else:
                newNode = self.parseLineGeneric(line)
                if newNode and 'name' in newNode and newNode['name'] == 'Histogram':
                    # if we encounter the histogram node, we turn on histogram parsing mode
                    hm = True
                    # and we store the level for all the upcoming histogram lines that follow
                    lh = 1 + newNode['level']

            if newNode:
                lc = newNode['level']

                # dispose of the 'level' attribute, we only need that information here
                if 'level' in newNode:
                    lc = newNode['level']
                    del newNode['level']

                # set parent
                newNode['parent'] = stack[lc-1]
                # add the node as a child of its immediate parent
                (stack[lc-1])['children'].append(newNode)
                # put the node on the stack, this will subsequently be used
                # by the the next iterations of this loop(if this node has children)
                stack[lc] = newNode
                # update the previous level
                lp = lc

        # store the tree in the class attribute for later use
        self.Data = root

    ## Group multiple options with the same
    ## prefix before the colon into a new parent
    ## having the common prefix as the name
    ##
    ## an example to illustrate this:
    ## <x>
    ##

    ##

    ## </x>
    ##
    ## =>
    ##
    ## <x>
    ##

    ##
    ##
    ##

    ## </x>
    def treeTransformGroup(self):
        root = self.Data
        stack = []
        stack.append(root)

        while len(stack) > 0:
            # We're visiting the next immediate node on the stack
            # (regular DFS traversal)
            x = stack.pop()
            if 'children' in x:
                y = x['children']
                i = 0
                ## a's keys will be common prefixes
                ## and the values will be new parent nodes which hold the
                ## children that will be transfered from x
                a = {}
                while i < len(y):
                    z = y[i]
                    match = re.match(self.RE_GROUPED_ENTRY, z.get('name',''), re.UNICODE)
                    if match:
                        d = match.groupdict()
                        dPrefix = d['prefix']
                        dName = d['name']
                        if dName == '':
                            del y[i]
                            continue
                        p = a.setdefault(dPrefix, \
                                {
                                    'children': [],
                                    'name': dPrefix,
                                    'value': '',
                                    'parent': x,
                                })
                        # update z's parent because it has been moved.
                        # to illustrate this, here is how the hierarchy changes:
                        # x->z => x->p->z
                        # so p is the new parent of x
                        z['parent'] = p
                        z['name'] = dName
                        # add z to p's children
                        p['children'] += [z]
                        del y[i]
                    else:
                        i += 1
                # at this point, depending on whether there were children to be grouped
                # they were(into a), and those that couldn't be will have stayed the same(in x['children']).
                # at this point, some nodes (some children of x) have been displaced and are now
                # children of new parent nodes, we are now expressing the fact that x has, among other children,
                # these new parent children that were created
                x['children'] += a.values()

            # Get x's children and put them on the stack
            # (continue the regular DFS)
            if 'children' in x:
                stack += x['children']

    # Note: this is only to be used for JSON serialization
    def treeTransformCompact(self, x):
        # this is basically a rebuilding of the tree in a more compact form.
        # here is a summary of what this method does:
        #
        # 1) we transform the tree as follows:
        # we aim to replace the 'children' attribute with either
        # an array or a dictionary, as follows:
        #
        # - all children have different names => we can store them in a dict
        # - if at least two children have the same name => we need to store them in an array
        #
        # 2) if a node has no children, and it has no additional attributes, then it
        # can be expressed as {k: v}
        #
        # check if it has no children
        xHasNoChildren = ('children' not in x) or ('children' in x and len(x['children']) == 0)
        # check if it has only basic properties: name,value,parent,children
        xHasOnlyNameValue = ('name' in x and 'value' in x and len(x.keys()) <= 4)

        # strip tree of parent attributes in order to avoid
        # circular references when serializing
        # to json
        del x['parent']

        if xHasNoChildren:
            del x['children']

            # x has no children
            if xHasOnlyNameValue:
                k = x['name']
                v = x['value']
                return [1,k,v]
            else:
                xname = x['name']
                xvalue = x['value']
                del x['name']
                del x['value']
                x['_value'] = xvalue
                return [2,xname,x]
        else:
            c = []
            xname = x['name']
            xvalue = x['value']
            # has children, recurse into children
            i = 0
            while i < len(x['children']):
                yi = x['children'][i]
                zi = self.treeTransformCompact(yi)
                c.append(zi)
                i += 1

            # constructing the new node
            w = None
            cnames = map(lambda z: z[1], c)
            if len(set(cnames)) == len(cnames):
                # the children all have distinct names, so w will be a dict
                w = {}
                for z in c:
                    w[z[1]] = z[2]
            else:
                # name collision are present, we need an array
                w = []
                for z in c:
                    w.append({z[1]: z[2]})

            return [3,xname,w]

    def serializeXML(self,root,xmlRoot):
        name = root['name']
        value = root['value']
        name = self.normalizeName(name)
        name = name.decode('utf-8')
        value = value.decode('utf-8')

        # serialize the node
        if 'children' in root and len(root['children']) > 0:
            for c in root['children']:
                cName = self.normalizeName(c['name'])
                xmlChild = SubElement(xmlRoot,cName)
                self.serializeXML(c,xmlChild)
        else:
            if name == self.HISTOGRAM_ELEM:
                xmlRoot.set('n', root['count'])
                xmlRoot.tag = self.HISTOGRAM_ELEM
                for k,v in root.iteritems():
                    # guard against undefined values(these are coming from the captures
                    # in the RE_LINE_HISTO regex, and the xml module will throw exceptions on
                    # the undefined values, so we want to avoid that)
                    # and check that the key is not an internal data key
                    if v and k not in ['name','value','parent','children']:
                        xmlRoot.set(k,v)
            else:
                xmlRoot.text=value

    def toJSON(self):
        Data = self.Data.copy()
        # run transformation to compact tree
        Data = self.treeTransformCompact(Data)
        # serialize to json
        return json.dumps(Data[2], indent=2)

    def toXML(self):
        Data = self.Data.copy()
        root = Data['children'][0]
        # serialize the root node and return it
        tree = ElementTree(Element('Image'))
        tree.getroot().set('file',Data['children'][0]['value'])
        self.serializeXML(root, tree.getroot())
        # prettify XML and return it
        unformattedXML = tostring(tree.getroot(),encoding='utf8')
        reparsedXML = minidom.parseString(unformattedXML)
        return reparsedXML.toprettyxml(indent="  ")

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='DICOM image metadata convertor')
    parser.add_argument("-i", dest="filename", required=True, help="input file")
    parser.add_argument('--raw' , '-r', dest='raw', action='store_true')
    parser.add_argument('--json', '-j', dest='json', action='store_true')
    parser.add_argument('--xml' , '-x', dest='xml', action='store_true')
    args = parser.parse_args()
    o = ImageMagickIdentifyParser()
    o.parse(args.filename)
    if args.raw:
        print o.Data
    if args.json:
        print o.toJSON()
    if args.xml:
        print o.toXML()
<|endoftext|>


<commit_before>import settings
import logging
import struct
from ingest import Ingest
from appAux import loadFile
import pyregf
import settings
import re
from ShimCacheParser_ACP import read_from_hive, write_it

logger = logging.getLogger(__name__)
# Module to ingest AppCompat data
# File name must start with "SYSTEM"
# Host name is extracted from the actual hive

class Appcompat_Raw_hive(Ingest):
    ingest_type = "appcompat_raw_hive"
    file_name_filter = "(?:.*)(?:\/|\\\)SYSTEM.*$"

    def __init__(self):
        super(Appcompat_Raw_hive, self).__init__()

    def getHostName(self, file_name_fullpath):
        file_object = loadFile(file_name_fullpath)
        regf_file = pyregf.file()
        regf_file.open_file_object(file_object, "r")
        # Get control set number
        tmp_key = regf_file.get_key_by_path(r'Select')
        if tmp_key is not None:
            controlset_number = tmp_key.get_value_by_name('Current').get_data_as_integer()
            # Get host name
            tmp_key = regf_file.get_key_by_path(r'ControlSet00' + str(controlset_number) + '\Control\ComputerName\ComputerName')
            host_name = tmp_key.get_value_by_name('ComputerName').get_data_as_string()
        else:
            # todo: Close everything down elegantly
            logger.error("Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin: %s" % file_name_fullpath)
            raise(Exception('Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin'))

        # Need to close these or the memory will never get freed:
        regf_file.close()
        del regf_file
        file_object.close()
        del file_object
        return host_name

    def checkMagic(self, file_name_fullpath):
        magic_ok = False
        # Check magic
        magic_id = self.id_filename(file_name_fullpath)
        if 'registry' in magic_id:
            file_object = loadFile(file_name_fullpath)
            regf_file = pyregf.file()
            regf_file.open_file_object(file_object, "r")
            magic_key = regf_file.get_key_by_path(r'Select')
            regf_file.close()
            del regf_file
            if magic_key is not None:
                magic_ok = True

            # Need to close these or the memory will never get freed:
            file_object.close()
            del file_object

        return magic_ok

    def calculateID(self, file_name_fullpath):
        instanceID = 0
        file_object = loadFile(file_name_fullpath)
        regf_file = pyregf.file()
        regf_file.open_file_object(file_object, "r")

        # Search for key containing ShimCache entries on all control sets
        # Use last modification time of the last modified one as instanceID
        root = regf_file.get_root_key()
        num_keys = root.get_number_of_sub_keys()
        for i in xrange(0,num_keys):
            tmp_key = root.get_sub_key(i)
            if "controlset" in tmp_key.get_name().lower():
                session_man_key = regf_file.get_key_by_path("%s\Control\Session Manager" % tmp_key.get_name())
                num_keys = session_man_key.get_number_of_sub_keys()
                for i in xrange(0, num_keys):
                    tmp_key = session_man_key.get_sub_key(i)
                    if "appcompatibility" in tmp_key.get_name().lower() or "appcompatcache" in tmp_key.get_name().lower():
                        last_write_time = tmp_key.get_last_written_time_as_integer()
                        if last_write_time > instanceID: instanceID = last_write_time
                        break

        # Need to close these or the memory will never get freed:
        regf_file.close()
        del regf_file
        file_object.close()
        del file_object
        return instanceID

    def processFile(self, file_fullpath, hostID, instanceID, rowsData):
        rowNumber = 0
        entries = None
        # Process file using ShimCacheParser
        try:
            entries = read_from_hive(file_fullpath, True)
            if not entries:
                logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath)
                return False
            else:
                rows = write_it(entries, "StringIO")[1:]
        except IOError, err:
            logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err))

        # Process records
        appCompatREGEX = re.compile(
            "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)")
        assert (rows is not None)
        for r in rows:
            if b'\x00' in r:
                logger.debug("NULL byte found, skipping bad shimcache parse: %s" % r)
                continue
            m = appCompatREGEX.match(r)
            if m:
                namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__, RowNumber=rowNumber,
                                                  LastModified=unicode(m.group(1)), LastUpdate=unicode(m.group(2)),
                                                  FilePath=unicode(m.group(3)),
                                                  FileName=unicode(m.group(4)), Size=unicode(m.group(5)),
                                                  ExecFlag=str(m.group(6)), InstanceID=instanceID)
                rowsData.append(namedrow)
                rowNumber += 1
            else:
                logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
<commit_msg>Datetime required from ingest modules now, not strings.<commit_after>import settings
import logging
import struct
from ingest import Ingest
from appAux import loadFile
import pyregf
import settings
import re
from datetime import datetime
from ShimCacheParser_ACP import read_from_hive, write_it

logger = logging.getLogger(__name__)
# Module to ingest AppCompat data
# File name must start with "SYSTEM"
# Host name is extracted from the actual hive

class Appcompat_Raw_hive(Ingest):
    ingest_type = "appcompat_raw_hive"
    file_name_filter = "(?:.*)(?:\/|\\\)SYSTEM.*$"

    def __init__(self):
        super(Appcompat_Raw_hive, self).__init__()

    def getHostName(self, file_name_fullpath):
        file_object = loadFile(file_name_fullpath)
        regf_file = pyregf.file()
        regf_file.open_file_object(file_object, "r")
        # Get control set number
        tmp_key = regf_file.get_key_by_path(r'Select')
        if tmp_key is not None:
            controlset_number = tmp_key.get_value_by_name('Current').get_data_as_integer()
            # Get host name
            tmp_key = regf_file.get_key_by_path(r'ControlSet00' + str(controlset_number) + '\Control\ComputerName\ComputerName')
            host_name = tmp_key.get_value_by_name('ComputerName').get_data_as_string()
        else:
            # todo: Close everything down elegantly
            logger.error("Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin: %s" % file_name_fullpath)
            raise(Exception('Attempting to process non-SYSTEM hive with appcompat_raw_hive plugin'))

        # Need to close these or the memory will never get freed:
        regf_file.close()
        del regf_file
        file_object.close()
        del file_object
        return host_name

    def checkMagic(self, file_name_fullpath):
        magic_ok = False
        # Check magic
        magic_id = self.id_filename(file_name_fullpath)
        if 'registry' in magic_id:
            file_object = loadFile(file_name_fullpath)
            regf_file = pyregf.file()
            regf_file.open_file_object(file_object, "r")
            magic_key = regf_file.get_key_by_path(r'Select')
            regf_file.close()
            del regf_file
            if magic_key is not None:
                magic_ok = True

            # Need to close these or the memory will never get freed:
            file_object.close()
            del file_object

        return magic_ok

    def calculateID(self, file_name_fullpath):
        instanceID = 0
        file_object = loadFile(file_name_fullpath)
        regf_file = pyregf.file()
        regf_file.open_file_object(file_object, "r")

        # Search for key containing ShimCache entries on all control sets
        # Use last modification time of the last modified one as instanceID
        root = regf_file.get_root_key()
        num_keys = root.get_number_of_sub_keys()
        for i in xrange(0,num_keys):
            tmp_key = root.get_sub_key(i)
            if "controlset" in tmp_key.get_name().lower():
                session_man_key = regf_file.get_key_by_path("%s\Control\Session Manager" % tmp_key.get_name())
                num_keys = session_man_key.get_number_of_sub_keys()
                for i in xrange(0, num_keys):
                    tmp_key = session_man_key.get_sub_key(i)
                    if "appcompatibility" in tmp_key.get_name().lower() or "appcompatcache" in tmp_key.get_name().lower():
                        last_write_time = tmp_key.get_last_written_time_as_integer()
                        if last_write_time > instanceID: instanceID = last_write_time
                        break

        # Need to close these or the memory will never get freed:
        regf_file.close()
        del regf_file
        file_object.close()
        del file_object
        return instanceID

    def processFile(self, file_fullpath, hostID, instanceID, rowsData):
        rowNumber = 0
        entries = None
        minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0)
        maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0)

        # Process file using ShimCacheParser
        try:
            entries = read_from_hive(file_fullpath, True)
            if not entries:
                logger.warning("[ShimCacheParser] found no entries for %s" % file_fullpath)
                return False
            else:
                rows = write_it(entries, "StringIO")[1:]
        except IOError, err:
            logger.error("[ShimCacheParser] Error opening binary file: %s" % str(err))

        # Process records
        appCompatREGEX = re.compile(
            "((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ]((?:\d\d\d\d\-\d\d\-\d\d \d\d:\d\d:\d\d)|N\/A)[, ](.*)\\\([^\\\]*)[, ](N\/A|\d*)[, ](N\/A|True|False)")
        assert (rows is not None)
        for r in rows:
            if b'\x00' in r:
                logger.debug("NULL byte found, skipping bad shimcache parse: %s" % r)
                continue
            m = appCompatREGEX.match(r)
            if m:
                try:
                    # Convert to timestamps:
                    if m.group(1) != 'N/A':
                        tmp_LastModified = datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S")
                    else: tmp_LastModified = minSQLiteDTS
                    if m.group(2) != 'N/A':
                        tmp_LastUpdate = datetime.strptime(m.group(2), "%Y-%m-%d %H:%M:%S")
                    else: tmp_LastUpdate = minSQLiteDTS

                except Exception as e:
                    print("crap")
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                    logger.info("Exception processing row (%s): %s [%s / %s / %s]" % (
                    e.message, unicode(ntpath.split(row[2])[0]), exc_type, fname, exc_tb.tb_lineno))

                namedrow = settings.EntriesFields(HostID=hostID, EntryType=settings.__APPCOMPAT__,
                                                  RowNumber=rowNumber,
                                                  LastModified=tmp_LastModified,
                                                  LastUpdate=tmp_LastUpdate,
                                                  FilePath=unicode(m.group(3)),
                                                  FileName=unicode(m.group(4)),
                                                  Size=unicode(m.group(5)),
                                                  ExecFlag=str(m.group(6)),
                                                  InstanceID=instanceID)
                rowsData.append(namedrow)
                rowNumber += 1
            else:
                logger.warning("Entry regex failed for: %s - %s" % (hostID, r))
<|endoftext|>