<commit_before>from functools import partial

import param
import numpy as np
import pandas as pd
import holoviews as hv
import datashader as ds
import colorcet as cc

from param import ParameterizedFunction, ParamOverrides
from holoviews.core.operation import Operation
from holoviews.streams import Stream, BoundsXY, LinkedStream
from holoviews.streams import Bounds as BoundsXY
from holoviews.plotting.bokeh.callbacks import Callback
from holoviews.operation.datashader import datashade, dynspread

from bokeh.palettes import Greys9

class filterpoints(hv.Operation):

    groupby = param.Parameter(default=None, doc="""
        Groupby argument to pass to Points creation.""")

    filter_range = param.Parameter(default={}, doc="""
        Dictionary of filter bounds.""")

    xdim = param.String(default='x')
    ydim = param.String(default='y')

    def _process(self, dset, key=None):

        if self.p.filter_range is not None:
            dset = dset.select(**self.p.filter_range)

        vdims = [dim for dim in dset.dimensions() if dim.name not in [self.p.xdim, self.p.ydim]]
        pts = dset.to(hv.Points, kdims=[self.p.xdim, self.p.ydim],
                        vdims=vdims,
                        groupby=self.p.groupby)

        return pts

# Define Stream class that stores filters for various Dimensions
class FilterStream(Stream):
    """
    Stream to apply arbitrary filtering on a Dataset.
    """

    filter_range = param.Dict(default={})

class ResetCallback(Callback):

    models = ['plot']
    on_events = ['reset']

class Reset(LinkedStream):
    def __init__(self, *args, **params):
        super(Reset, self).__init__(self, *args, **dict(params, transient=True))

Stream._callbacks['bokeh'][Reset] = ResetCallback

class filter_dset(Operation):
    filter_range = param.Dict(default={}, doc="""
        Dictionary of filter bounds.""")

    def _process(self, dset, key=None):
        if self.p.filter_range is not None:
            dset = dset.select(**self.p.filter_range)
        return dset

# Define Operation that filters based on FilterStream state (which provides the filter_range)
class filterpoints(Operation):

    filter_range = param.Dict(default={}, doc="""
        Dictionary of filter bounds.""")
    xdim = param.String(default='x')
    ydim = param.String(default='y')
    set_title = param.Boolean(default=True)

    def _process(self, dset, key=None):
        dset = filter_dset(dset, filter_range=self.p.filter_range)
        kdims = [dset.get_dimension(self.p.xdim), dset.get_dimension(self.p.ydim)]
        vdims = [dim for dim in dset.dimensions() if dim.name not in kdims]
        pts = hv.Points(dset, kdims=kdims, vdims=vdims)
        return pts
        if self.p.set_title:
            ydata = dset.data[self.p.ydim]
            title = 'mean = {:.3f}, std = {:.3f} ({:.0f})'.format(ydata.mean(),
                                                                  ydata.std(),
                                                                  len(ydata))
            pts = pts.relabel(title)
        return pts

def notify_stream(bounds, filter_stream, xdim, ydim):
    """
    Function to attach to bounds stream as subscriber to notify FilterStream.
    """
    l, b, r, t = bounds
    filter_range = dict(filter_stream.filter_range)
    for dim, (low, high) in [(xdim, (l, r)), (ydim, (b, t))]:
        ## If you want to take the intersection of x selections, e.g.
        # if dim in filter_range:
        #     old_low, old_high = filter_range[dim]
        #     filter_range[dim]= (max(old_low, low), min(old_high, high))
        # else:
        #     filter_range[dim] = (low, high)
        filter_range[dim] = (low, high)
    filter_stream.event(filter_range=filter_range)

def reset_stream(filter_stream):
    filter_stream.event(filter_range={})

class scattersky(ParameterizedFunction):
    """
    Creates two datashaded views from a Dataset.
    """

    xdim = param.String(default='x')
    ydim = param.String(default='y0')
    scatter_cmap = param.String(default='fire')
    sky_cmap = param.String(default='coolwarm')
    height = param.Number(default=300)
    width = param.Number(default=900)
    filter_stream = param.ClassSelector(default=FilterStream(), class_=FilterStream)
    show_rawsky = param.Boolean(default=False)

    def __call__(self, dset, **params):
        self.p = ParamOverrides(self, params)
        if self.p.ydim not in dset.dimensions():
            raise ValueError('{} not in Dataset.'.format(self.p.ydim))

        # Set up scatter plot
        scatter_filterpoints = filterpoints.instance(xdim=self.p.xdim, ydim=self.p.ydim)
        scatter_pts = hv.util.Dynamic(dset, operation=scatter_filterpoints,
                                      streams=[self.p.filter_stream])
        scatter_opts = dict(plot={'height':self.p.height, 'width':self.p.width - self.p.height,
                                  'tools':['box_select']},
                           norm=dict(axiswise=True))
        scatter_shaded = datashade(scatter_pts, cmap=cc.palette[self.p.scatter_cmap])
        scatter = dynspread(scatter_shaded).opts(**scatter_opts)

        # Set up sky plot
        sky_filterpoints = filterpoints.instance(xdim='ra', ydim='dec', set_title=False)
        sky_pts = hv.util.Dynamic(dset, operation=sky_filterpoints,
                                  streams=[self.p.filter_stream])
        sky_opts = dict(plot={'height':self.p.height, 'width':self.p.height,
                              'tools':['box_select']},
                        norm=dict(axiswise=True))
        sky_shaded = datashade(sky_pts, cmap=cc.palette[self.p.sky_cmap],
                               aggregator=ds.mean(self.p.ydim), height=self.p.height,
                               width=self.p.width)
        sky = dynspread(sky_shaded).opts(**sky_opts)

        # Set up BoundsXY streams to listen to box_select events and notify FilterStream
        scatter_select = BoundsXY(source=scatter)
        scatter_notifier = partial(notify_stream, filter_stream=self.p.filter_stream,
                                   xdim=self.p.xdim, ydim=self.p.ydim)
        scatter_select.add_subscriber(scatter_notifier)

        sky_select = BoundsXY(source=sky)
        sky_notifier = partial(notify_stream, filter_stream=self.p.filter_stream,
                               xdim='ra', ydim='dec')
        sky_select.add_subscriber(sky_notifier)

        # Reset
        reset = Reset(source=scatter)
        reset.add_subscriber(partial(reset_stream, self.p.filter_stream))

        raw_scatter = datashade(scatter_filterpoints(dset), cmap=Greys9[::-1][:5])
        if self.p.show_rawsky:
            raw_sky = datashade(sky_filterpoints(dset), cmap=Greys9[::-1][:5])
            return raw_scatter*scatter + raw_sky*sky

        else:
            return raw_scatter*scatter + sky

class multi_scattersky(ParameterizedFunction):

    filter_stream = param.ClassSelector(default=FilterStream(), class_=FilterStream)

    ignored_dimensions = param.List(default=['x', 'ra', 'dec', 'label'])

    def _get_ydims(self, dset):
        # Get dimensions from first Dataset type found in input
        return [dim.name for dim in dset.traverse(lambda x: x, [hv.Dataset])[0].dimensions()
                if dim.name not in self.p.ignored_dimensions]

    def __call__(self, dset, **params):
        self.p = param.ParamOverrides(self, params)
        return hv.Layout([scattersky(dset, filter_stream=self.p.filter_stream,
                                  ydim=ydim)
                       for ydim in self._get_ydims(dset)]).cols(2)
<commit_msg>put title back in<commit_after>from functools import partial

import param
import numpy as np
import pandas as pd
import holoviews as hv
import datashader as ds
import colorcet as cc

from param import ParameterizedFunction, ParamOverrides
from holoviews.core.operation import Operation
from holoviews.streams import Stream, BoundsXY, LinkedStream
from holoviews.streams import Bounds as BoundsXY
from holoviews.plotting.bokeh.callbacks import Callback
from holoviews.operation.datashader import datashade, dynspread

from bokeh.palettes import Greys9

class filterpoints(hv.Operation):

    groupby = param.Parameter(default=None, doc="""
        Groupby argument to pass to Points creation.""")

    filter_range = param.Parameter(default={}, doc="""
        Dictionary of filter bounds.""")

    xdim = param.String(default='x')
    ydim = param.String(default='y')

    def _process(self, dset, key=None):

        if self.p.filter_range is not None:
            dset = dset.select(**self.p.filter_range)

        vdims = [dim for dim in dset.dimensions() if dim.name not in [self.p.xdim, self.p.ydim]]
        pts = dset.to(hv.Points, kdims=[self.p.xdim, self.p.ydim],
                        vdims=vdims,
                        groupby=self.p.groupby)

        return pts

# Define Stream class that stores filters for various Dimensions
class FilterStream(Stream):
    """
    Stream to apply arbitrary filtering on a Dataset.
    """

    filter_range = param.Dict(default={})

class ResetCallback(Callback):

    models = ['plot']
    on_events = ['reset']

class Reset(LinkedStream):
    def __init__(self, *args, **params):
        super(Reset, self).__init__(self, *args, **dict(params, transient=True))

Stream._callbacks['bokeh'][Reset] = ResetCallback

class filter_dset(Operation):
    filter_range = param.Dict(default={}, doc="""
        Dictionary of filter bounds.""")

    def _process(self, dset, key=None):
        if self.p.filter_range is not None:
            dset = dset.select(**self.p.filter_range)
        return dset

# Define Operation that filters based on FilterStream state (which provides the filter_range)
class filterpoints(Operation):

    filter_range = param.Dict(default={}, doc="""
        Dictionary of filter bounds.""")
    xdim = param.String(default='x')
    ydim = param.String(default='y')
    set_title = param.Boolean(default=True)

    def _process(self, dset, key=None):
        dset = filter_dset(dset, filter_range=self.p.filter_range)
        kdims = [dset.get_dimension(self.p.xdim), dset.get_dimension(self.p.ydim)]
        vdims = [dim for dim in dset.dimensions() if dim.name not in kdims]
        pts = hv.Points(dset, kdims=kdims, vdims=vdims)
        if self.p.set_title:
            ydata = dset.data[self.p.ydim]
            title = 'mean = {:.3f}, std = {:.3f} ({:.0f})'.format(ydata.mean(),
                                                                  ydata.std(),
                                                                  len(ydata))
            pts = pts.relabel(title)
        return pts

def notify_stream(bounds, filter_stream, xdim, ydim):
    """
    Function to attach to bounds stream as subscriber to notify FilterStream.
    """
    l, b, r, t = bounds
    filter_range = dict(filter_stream.filter_range)
    for dim, (low, high) in [(xdim, (l, r)), (ydim, (b, t))]:
        ## If you want to take the intersection of x selections, e.g.
        # if dim in filter_range:
        #     old_low, old_high = filter_range[dim]
        #     filter_range[dim]= (max(old_low, low), min(old_high, high))
        # else:
        #     filter_range[dim] = (low, high)
        filter_range[dim] = (low, high)
    filter_stream.event(filter_range=filter_range)

def reset_stream(filter_stream):
    filter_stream.event(filter_range={})

class scattersky(ParameterizedFunction):
    """
    Creates two datashaded views from a Dataset.
    """

    xdim = param.String(default='x')
    ydim = param.String(default='y0')
    scatter_cmap = param.String(default='fire')
    sky_cmap = param.String(default='coolwarm')
    height = param.Number(default=300)
    width = param.Number(default=900)
    filter_stream = param.ClassSelector(default=FilterStream(), class_=FilterStream)
    show_rawsky = param.Boolean(default=False)

    def __call__(self, dset, **params):
        self.p = ParamOverrides(self, params)
        if self.p.ydim not in dset.dimensions():
            raise ValueError('{} not in Dataset.'.format(self.p.ydim))

        # Set up scatter plot
        scatter_filterpoints = filterpoints.instance(xdim=self.p.xdim, ydim=self.p.ydim)
        scatter_pts = hv.util.Dynamic(dset, operation=scatter_filterpoints,
                                      streams=[self.p.filter_stream])
        scatter_opts = dict(plot={'height':self.p.height, 'width':self.p.width - self.p.height,
                                  'tools':['box_select']},
                           norm=dict(axiswise=True))
        scatter_shaded = datashade(scatter_pts, cmap=cc.palette[self.p.scatter_cmap])
        scatter = dynspread(scatter_shaded).opts(**scatter_opts)

        # Set up sky plot
        sky_filterpoints = filterpoints.instance(xdim='ra', ydim='dec', set_title=False)
        sky_pts = hv.util.Dynamic(dset, operation=sky_filterpoints,
                                  streams=[self.p.filter_stream])
        sky_opts = dict(plot={'height':self.p.height, 'width':self.p.height,
                              'tools':['box_select']},
                        norm=dict(axiswise=True))
        sky_shaded = datashade(sky_pts, cmap=cc.palette[self.p.sky_cmap],
                               aggregator=ds.mean(self.p.ydim), height=self.p.height,
                               width=self.p.width)
        sky = dynspread(sky_shaded).opts(**sky_opts)

        # Set up BoundsXY streams to listen to box_select events and notify FilterStream
        scatter_select = BoundsXY(source=scatter)
        scatter_notifier = partial(notify_stream, filter_stream=self.p.filter_stream,
                                   xdim=self.p.xdim, ydim=self.p.ydim)
        scatter_select.add_subscriber(scatter_notifier)

        sky_select = BoundsXY(source=sky)
        sky_notifier = partial(notify_stream, filter_stream=self.p.filter_stream,
                               xdim='ra', ydim='dec')
        sky_select.add_subscriber(sky_notifier)

        # Reset
        reset = Reset(source=scatter)
        reset.add_subscriber(partial(reset_stream, self.p.filter_stream))

        raw_scatter = datashade(scatter_filterpoints(dset), cmap=Greys9[::-1][:5])
        if self.p.show_rawsky:
            raw_sky = datashade(sky_filterpoints(dset), cmap=Greys9[::-1][:5])
            return raw_scatter*scatter + raw_sky*sky

        else:
            return raw_scatter*scatter + sky

class multi_scattersky(ParameterizedFunction):

    filter_stream = param.ClassSelector(default=FilterStream(), class_=FilterStream)

    ignored_dimensions = param.List(default=['x', 'ra', 'dec', 'label'])

    def _get_ydims(self, dset):
        # Get dimensions from first Dataset type found in input
        return [dim.name for dim in dset.traverse(lambda x: x, [hv.Dataset])[0].dimensions()
                if dim.name not in self.p.ignored_dimensions]

    def __call__(self, dset, **params):
        self.p = param.ParamOverrides(self, params)
        return hv.Layout([scattersky(dset, filter_stream=self.p.filter_stream,
                                  ydim=ydim)
                       for ydim in self._get_ydims(dset)]).cols(2)
<|endoftext|>


<commit_before>import ConfigParser
import logging
import re

from cStringIO import StringIO

from . import conf
from . import exc
from . import lsb
from .cliutil import priority

log = logging.getLogger(__name__)

def write_conf(cluster, conf):
    import os

    path = '/etc/ceph/{cluster}.conf'.format(cluster=cluster)
    tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid())

    with file(tmp, 'w') as f:
        f.write(conf)
        f.flush()
        os.fsync(f)
    os.rename(tmp, path)

def create_mon(cluster, monitor_keyring, init):
    import os
    import socket
    import subprocess

    hostname = socket.gethostname()
    path = '/var/lib/ceph/mon/ceph-{hostname}'.format(
        hostname=hostname,
        )
    done_path = '/var/lib/ceph/mon/ceph-{hostname}/done'.format(
        hostname=hostname,
        )
    init_path = '/var/lib/ceph/mon/ceph-{hostname}/{init}'.format(
        hostname=hostname,
        init=init,
        )

    if not os.path.exists(path):
        os.mkdir(path)

    if not os.path.exists(done_path):
        keyring = '/var/lib/ceph/tmp/{cluster}-{hostname}.mon.keyring'.format(
            cluster=cluster,
            hostname=hostname,
            )

        with file(keyring, 'w') as f:
            f.write(monitor_keyring)

        subprocess.check_call(
            args=[
                'ceph-mon',
                '--cluster', cluster,
                '--mkfs',
                '-i', hostname,
                '--keyring', keyring,
                ],
            )
        os.unlink(keyring)
        with file(done_path, 'w'):
            pass

    if not os.path.exists(init_path):
        with file(init_path, 'w'):
            pass

    if init == 'upstart':
        subprocess.check_call(
            args=[
                'initctl',
                'emit',
                'ceph-mon',
                'cluster={cluster}'.format(cluster=cluster),
                'id={hostname}'.format(hostname=hostname),
                ],
            )
    elif init == 'sysvinit':
        subprocess.check_call(
            args=[
                'service',
                'ceph',
                'start',
                'mon.{hostname}'.format(hostname=hostname),
                ],
            )

def mon_create(args):
    cfg = conf.load(args)
    if not args.mon:
        try:
            mon_initial_members = cfg.get('global', 'mon_initial_members')
        except (ConfigParser.NoSectionError,
                ConfigParser.NoOptionError):
            pass
        else:
            args.mon = re.split(r'[,\s]+', mon_initial_members)

    if not args.mon:
        raise exc.NeedHostError()

    try:
        with file('{cluster}.mon.keyring'.format(cluster=args.cluster),
                  'rb') as f:
            monitor_keyring = f.read()
    except IOError:
        raise RuntimeError('mon keyring not found; run \'new\' to create a new cluster')

    log.debug(
        'Deploying mon, cluster %s hosts %s',
        args.cluster,
        ' '.join(args.mon),
        )

    for hostname in args.mon:
        log.debug('Deploying mon to %s', hostname)

        # TODO username
        sudo = args.pushy('ssh+sudo:{hostname}'.format(hostname=hostname))

        lsb_release_r = sudo.compile(lsb.lsb_release)
        (distro, release, codename) = lsb_release_r()
        init = lsb.choose_init(distro, codename)
        log.debug('Distro %s codename %s, will use %s',
                  distro, codename, init)

        write_conf_r = sudo.compile(write_conf)
        conf_data = StringIO()
        cfg.write(conf_data)
        write_conf_r(
            cluster=args.cluster,
            conf=conf_data.getvalue(),
            )

        create_mon_r = sudo.compile(create_mon)
        create_mon_r(
            cluster=args.cluster,
            monitor_keyring=monitor_keyring,
            init=init,
            )

        # TODO add_bootstrap_peer_hint

def mon(args):
    if args.subcommand == 'create':
        mon_create(args)
    else:
        log.error('subcommand %s not implemented', args.subcommand)

@priority(30)
def make(parser):
    """
    Deploy ceph monitor on remote hosts.
    """
    parser.add_argument(
        'subcommand',
        metavar='SUBCOMMAND',
        choices=[
            'create',
            ],
        )
    parser.add_argument(
        'mon',
        metavar='HOST',
        nargs='*',
        help='host to deploy on',
        )
    parser.set_defaults(
        func=mon,
        )
<commit_msg>mon: help for subcommand<commit_after>import ConfigParser
import logging
import re

from cStringIO import StringIO

from . import conf
from . import exc
from . import lsb
from .cliutil import priority

log = logging.getLogger(__name__)

def write_conf(cluster, conf):
    import os

    path = '/etc/ceph/{cluster}.conf'.format(cluster=cluster)
    tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid())

    with file(tmp, 'w') as f:
        f.write(conf)
        f.flush()
        os.fsync(f)
    os.rename(tmp, path)

def create_mon(cluster, monitor_keyring, init):
    import os
    import socket
    import subprocess

    hostname = socket.gethostname()
    path = '/var/lib/ceph/mon/ceph-{hostname}'.format(
        hostname=hostname,
        )
    done_path = '/var/lib/ceph/mon/ceph-{hostname}/done'.format(
        hostname=hostname,
        )
    init_path = '/var/lib/ceph/mon/ceph-{hostname}/{init}'.format(
        hostname=hostname,
        init=init,
        )

    if not os.path.exists(path):
        os.mkdir(path)

    if not os.path.exists(done_path):
        keyring = '/var/lib/ceph/tmp/{cluster}-{hostname}.mon.keyring'.format(
            cluster=cluster,
            hostname=hostname,
            )

        with file(keyring, 'w') as f:
            f.write(monitor_keyring)

        subprocess.check_call(
            args=[
                'ceph-mon',
                '--cluster', cluster,
                '--mkfs',
                '-i', hostname,
                '--keyring', keyring,
                ],
            )
        os.unlink(keyring)
        with file(done_path, 'w'):
            pass

    if not os.path.exists(init_path):
        with file(init_path, 'w'):
            pass

    if init == 'upstart':
        subprocess.check_call(
            args=[
                'initctl',
                'emit',
                'ceph-mon',
                'cluster={cluster}'.format(cluster=cluster),
                'id={hostname}'.format(hostname=hostname),
                ],
            )
    elif init == 'sysvinit':
        subprocess.check_call(
            args=[
                'service',
                'ceph',
                'start',
                'mon.{hostname}'.format(hostname=hostname),
                ],
            )

def mon_create(args):
    cfg = conf.load(args)
    if not args.mon:
        try:
            mon_initial_members = cfg.get('global', 'mon_initial_members')
        except (ConfigParser.NoSectionError,
                ConfigParser.NoOptionError):
            pass
        else:
            args.mon = re.split(r'[,\s]+', mon_initial_members)

    if not args.mon:
        raise exc.NeedHostError()

    try:
        with file('{cluster}.mon.keyring'.format(cluster=args.cluster),
                  'rb') as f:
            monitor_keyring = f.read()
    except IOError:
        raise RuntimeError('mon keyring not found; run \'new\' to create a new cluster')

    log.debug(
        'Deploying mon, cluster %s hosts %s',
        args.cluster,
        ' '.join(args.mon),
        )

    for hostname in args.mon:
        log.debug('Deploying mon to %s', hostname)

        # TODO username
        sudo = args.pushy('ssh+sudo:{hostname}'.format(hostname=hostname))

        lsb_release_r = sudo.compile(lsb.lsb_release)
        (distro, release, codename) = lsb_release_r()
        init = lsb.choose_init(distro, codename)
        log.debug('Distro %s codename %s, will use %s',
                  distro, codename, init)

        write_conf_r = sudo.compile(write_conf)
        conf_data = StringIO()
        cfg.write(conf_data)
        write_conf_r(
            cluster=args.cluster,
            conf=conf_data.getvalue(),
            )

        create_mon_r = sudo.compile(create_mon)
        create_mon_r(
            cluster=args.cluster,
            monitor_keyring=monitor_keyring,
            init=init,
            )

        # TODO add_bootstrap_peer_hint

def mon(args):
    if args.subcommand == 'create':
        mon_create(args)
    else:
        log.error('subcommand %s not implemented', args.subcommand)

@priority(30)
def make(parser):
    """
    Deploy ceph monitor on remote hosts.
    """
    parser.add_argument(
        'subcommand',
        metavar='SUBCOMMAND',
        choices=[
            'create',
            'destroy',
            ],
        help='create or destroy',
        )
    parser.add_argument(
        'mon',
        metavar='HOST',
        nargs='*',
        help='host to deploy on',
        )
    parser.set_defaults(
        func=mon,
        )
<|endoftext|>


<commit_before>"""Tests of disco_elb"""
from unittest import TestCase
from mock import MagicMock
from moto import mock_elb
from disco_aws_automation import DiscoELB

TEST_ENV_NAME = 'unittestenv'
TEST_HOSTCLASS = 'mhcunit'
TEST_VPC_ID = 'vpc-56e10e3d'  # the hard coded VPC Id that moto will always return
TEST_DOMAIN_NAME = 'test.example.com'

def _get_vpc_mock():
    vpc_mock = MagicMock()
    vpc_mock.environment_name = TEST_ENV_NAME
    vpc_mock.vpc = MagicMock()
    vpc_mock.vpc.id = TEST_VPC_ID
    return vpc_mock

class DiscoELBTests(TestCase):
    """Test DiscoELB"""

    def setUp(self):
        self.disco_elb = DiscoELB(_get_vpc_mock(), route53=MagicMock(), acm=MagicMock(), iam=MagicMock())
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value="arn:aws:acm::123:blah")
        self.disco_elb.iam.get_certificate_arn = MagicMock(return_value="arn:aws:iam::123:blah")

    # pylint: disable=too-many-arguments
    def _create_elb(self, hostclass=None, public=False, tls=False,
                    instance_protocol='HTTP', instance_port=80,
                    elb_protocol='HTTP', elb_port=80,
                    idle_timeout=None, connection_draining_timeout=None,
                    sticky_app_cookie=None):
        return self.disco_elb.get_or_create_elb(
            hostclass=hostclass or TEST_HOSTCLASS,
            security_groups=['sec-1'],
            subnets=['sub-1'],
            hosted_zone_name=TEST_DOMAIN_NAME,
            health_check_url="/" if instance_protocol in ('HTTP', 'HTTPS') else "",
            instance_protocol=instance_protocol,
            instance_port=instance_port,
            elb_protocol="HTTPS" if tls else elb_protocol,
            elb_port=443 if tls else elb_port,
            elb_public=public,
            sticky_app_cookie=sticky_app_cookie,
            idle_timeout=idle_timeout,
            connection_draining_timeout=connection_draining_timeout)

    @mock_elb
    def test_get_certificate_arn_prefers_acm(self):
        '''get_certificate_arn() prefers an ACM provided certificate'''
        self.assertEqual(self.disco_elb.get_certificate_arn("dummy"), "arn:aws:acm::123:blah")

    @mock_elb
    def test_get_certificate_arn_fallback_to_iam(self):
        '''get_certificate_arn() uses an IAM certificate if no ACM cert available'''
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value=None)
        self.assertEqual(self.disco_elb.get_certificate_arn("dummy"), "arn:aws:iam::123:blah")

    @mock_elb
    def test_get_cname(self):
        '''Make sure get_cname returns what we expect'''
        self.assertEqual(self.disco_elb.get_cname(TEST_HOSTCLASS, TEST_DOMAIN_NAME),
                         "mhcunit-unittestenv.test.example.com")

    @mock_elb
    def test_get_elb_with_create(self):
        """Test creating a ELB"""
        self._create_elb()
        self.assertEquals(
            len(self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']), 1)

    @mock_elb
    def test_get_elb_with_update(self):
        """Updating an ELB doesn't add create a new ELB"""
        self._create_elb()
        self._create_elb()
        self.assertEquals(
            len(self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']), 1)

    @mock_elb
    def test_get_elb_internal(self):
        """Test creation an internal private ELB"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb()
        self.disco_elb.elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_internal_no_tls(self):
        """Test creation an internal private ELB"""
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value=None)
        self.disco_elb.iam.get_certificate_arn = MagicMock(return_value=None)
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb()
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': ''
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_external(self):
        """Test creation a publically accessible ELB"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(public=True)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'])

    @mock_elb
    def test_get_elb_with_tls(self):
        """Test creation an ELB with TLS"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(tls=True)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTPS',
                'LoadBalancerPort': 443,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_with_tcp(self):
        """Test creation an ELB with TCP"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(instance_protocol='TCP', instance_port=25,
                         elb_protocol='TCP', elb_port=25)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'TCP',
                'LoadBalancerPort': 25,
                'InstanceProtocol': 'TCP',
                'InstancePort': 25,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_with_idle_timeout(self):
        """Test creating an ELB with an idle timeout"""
        client = self.disco_elb.elb_client
        client.modify_load_balancer_attributes = MagicMock(wraps=client.modify_load_balancer_attributes)

        self._create_elb(idle_timeout=100)

        client.modify_load_balancer_attributes.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            LoadBalancerAttributes={'ConnectionDraining': {'Enabled': False, 'Timeout': 0},
                                    'ConnectionSettings': {'IdleTimeout': 100}}
        )

    @mock_elb
    def test_get_elb_with_connection_draining(self):
        """Test creating ELB with connection draining"""
        client = self.disco_elb.elb_client
        client.modify_load_balancer_attributes = MagicMock(wraps=client.modify_load_balancer_attributes)

        self._create_elb(connection_draining_timeout=100)

        client.modify_load_balancer_attributes.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            LoadBalancerAttributes={'ConnectionDraining': {'Enabled': True, 'Timeout': 100}}
        )

    @mock_elb
    def test_delete_elb(self):
        """Test deleting an ELB"""
        self._create_elb()
        self.disco_elb.delete_elb(TEST_HOSTCLASS)
        load_balancers = self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']
        self.assertEquals(len(load_balancers), 0)

    @mock_elb
    def test_get_existing_elb(self):
        """Test get_elb for a hostclass"""
        self._create_elb()
        self.assertIsNotNone(self.disco_elb.get_elb(TEST_HOSTCLASS))

    @mock_elb
    def test_list(self):
        """Test getting the list of ELBs"""
        self._create_elb(hostclass='mhcbar')
        self._create_elb(hostclass='mhcfoo')
        self.assertEquals(len(self.disco_elb.list()), 2)

    @mock_elb
    def test_elb_delete(self):
        """Test deletion of ELBs"""
        self._create_elb(hostclass='mhcbar')
        self.disco_elb.delete_elb(hostclass='mhcbar')
        self.assertEquals(len(self.disco_elb.list()), 0)

    @mock_elb
    def test_destroy_all_elbs(self):
        """Test deletion of all ELBs"""
        self._create_elb(hostclass='mhcbar')
        self._create_elb(hostclass='mhcfoo')
        self.disco_elb.destroy_all_elbs()
        self.assertEquals(len(self.disco_elb.list()), 0)
<commit_msg>Forced to uppercase when checking instance_protocol in unit test<commit_after>"""Tests of disco_elb"""
from unittest import TestCase
from mock import MagicMock
from moto import mock_elb
from disco_aws_automation import DiscoELB

TEST_ENV_NAME = 'unittestenv'
TEST_HOSTCLASS = 'mhcunit'
TEST_VPC_ID = 'vpc-56e10e3d'  # the hard coded VPC Id that moto will always return
TEST_DOMAIN_NAME = 'test.example.com'

def _get_vpc_mock():
    vpc_mock = MagicMock()
    vpc_mock.environment_name = TEST_ENV_NAME
    vpc_mock.vpc = MagicMock()
    vpc_mock.vpc.id = TEST_VPC_ID
    return vpc_mock

class DiscoELBTests(TestCase):
    """Test DiscoELB"""

    def setUp(self):
        self.disco_elb = DiscoELB(_get_vpc_mock(), route53=MagicMock(), acm=MagicMock(), iam=MagicMock())
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value="arn:aws:acm::123:blah")
        self.disco_elb.iam.get_certificate_arn = MagicMock(return_value="arn:aws:iam::123:blah")

    # pylint: disable=too-many-arguments
    def _create_elb(self, hostclass=None, public=False, tls=False,
                    instance_protocol='HTTP', instance_port=80,
                    elb_protocol='HTTP', elb_port=80,
                    idle_timeout=None, connection_draining_timeout=None,
                    sticky_app_cookie=None):
        return self.disco_elb.get_or_create_elb(
            hostclass=hostclass or TEST_HOSTCLASS,
            security_groups=['sec-1'],
            subnets=['sub-1'],
            hosted_zone_name=TEST_DOMAIN_NAME,
            health_check_url="/" if instance_protocol.upper() in ('HTTP', 'HTTPS') else "",
            instance_protocol=instance_protocol,
            instance_port=instance_port,
            elb_protocol="HTTPS" if tls else elb_protocol,
            elb_port=443 if tls else elb_port,
            elb_public=public,
            sticky_app_cookie=sticky_app_cookie,
            idle_timeout=idle_timeout,
            connection_draining_timeout=connection_draining_timeout)

    @mock_elb
    def test_get_certificate_arn_prefers_acm(self):
        '''get_certificate_arn() prefers an ACM provided certificate'''
        self.assertEqual(self.disco_elb.get_certificate_arn("dummy"), "arn:aws:acm::123:blah")

    @mock_elb
    def test_get_certificate_arn_fallback_to_iam(self):
        '''get_certificate_arn() uses an IAM certificate if no ACM cert available'''
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value=None)
        self.assertEqual(self.disco_elb.get_certificate_arn("dummy"), "arn:aws:iam::123:blah")

    @mock_elb
    def test_get_cname(self):
        '''Make sure get_cname returns what we expect'''
        self.assertEqual(self.disco_elb.get_cname(TEST_HOSTCLASS, TEST_DOMAIN_NAME),
                         "mhcunit-unittestenv.test.example.com")

    @mock_elb
    def test_get_elb_with_create(self):
        """Test creating a ELB"""
        self._create_elb()
        self.assertEquals(
            len(self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']), 1)

    @mock_elb
    def test_get_elb_with_update(self):
        """Updating an ELB doesn't add create a new ELB"""
        self._create_elb()
        self._create_elb()
        self.assertEquals(
            len(self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']), 1)

    @mock_elb
    def test_get_elb_internal(self):
        """Test creation an internal private ELB"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb()
        self.disco_elb.elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_internal_no_tls(self):
        """Test creation an internal private ELB"""
        self.disco_elb.acm.get_certificate_arn = MagicMock(return_value=None)
        self.disco_elb.iam.get_certificate_arn = MagicMock(return_value=None)
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb()
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': ''
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_external(self):
        """Test creation a publically accessible ELB"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(public=True)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTP',
                'LoadBalancerPort': 80,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'])

    @mock_elb
    def test_get_elb_with_tls(self):
        """Test creation an ELB with TLS"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(tls=True)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'HTTPS',
                'LoadBalancerPort': 443,
                'InstanceProtocol': 'HTTP',
                'InstancePort': 80,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_with_tcp(self):
        """Test creation an ELB with TCP"""
        elb_client = self.disco_elb.elb_client
        elb_client.create_load_balancer = MagicMock(wraps=elb_client.create_load_balancer)
        self._create_elb(instance_protocol='TCP', instance_port=25,
                         elb_protocol='TCP', elb_port=25)
        elb_client.create_load_balancer.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            Listeners=[{
                'Protocol': 'TCP',
                'LoadBalancerPort': 25,
                'InstanceProtocol': 'TCP',
                'InstancePort': 25,
                'SSLCertificateId': 'arn:aws:acm::123:blah'
            }],
            Subnets=['sub-1'],
            SecurityGroups=['sec-1'],
            Scheme='internal')

    @mock_elb
    def test_get_elb_with_idle_timeout(self):
        """Test creating an ELB with an idle timeout"""
        client = self.disco_elb.elb_client
        client.modify_load_balancer_attributes = MagicMock(wraps=client.modify_load_balancer_attributes)

        self._create_elb(idle_timeout=100)

        client.modify_load_balancer_attributes.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            LoadBalancerAttributes={'ConnectionDraining': {'Enabled': False, 'Timeout': 0},
                                    'ConnectionSettings': {'IdleTimeout': 100}}
        )

    @mock_elb
    def test_get_elb_with_connection_draining(self):
        """Test creating ELB with connection draining"""
        client = self.disco_elb.elb_client
        client.modify_load_balancer_attributes = MagicMock(wraps=client.modify_load_balancer_attributes)

        self._create_elb(connection_draining_timeout=100)

        client.modify_load_balancer_attributes.assert_called_once_with(
            LoadBalancerName='unittestenv-mhcunit',
            LoadBalancerAttributes={'ConnectionDraining': {'Enabled': True, 'Timeout': 100}}
        )

    @mock_elb
    def test_delete_elb(self):
        """Test deleting an ELB"""
        self._create_elb()
        self.disco_elb.delete_elb(TEST_HOSTCLASS)
        load_balancers = self.disco_elb.elb_client.describe_load_balancers()['LoadBalancerDescriptions']
        self.assertEquals(len(load_balancers), 0)

    @mock_elb
    def test_get_existing_elb(self):
        """Test get_elb for a hostclass"""
        self._create_elb()
        self.assertIsNotNone(self.disco_elb.get_elb(TEST_HOSTCLASS))

    @mock_elb
    def test_list(self):
        """Test getting the list of ELBs"""
        self._create_elb(hostclass='mhcbar')
        self._create_elb(hostclass='mhcfoo')
        self.assertEquals(len(self.disco_elb.list()), 2)

    @mock_elb
    def test_elb_delete(self):
        """Test deletion of ELBs"""
        self._create_elb(hostclass='mhcbar')
        self.disco_elb.delete_elb(hostclass='mhcbar')
        self.assertEquals(len(self.disco_elb.list()), 0)

    @mock_elb
    def test_destroy_all_elbs(self):
        """Test deletion of all ELBs"""
        self._create_elb(hostclass='mhcbar')
        self._create_elb(hostclass='mhcfoo')
        self.disco_elb.destroy_all_elbs()
        self.assertEquals(len(self.disco_elb.list()), 0)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- mode: python; coding: utf-8 -*-
# Copyright 2018 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""script to write M&C records to a CSV file

"""
from __future__ import absolute_import, division, print_function

import sys
from astropy.time import Time

from hera_mc import mc, cm_utils

valid_tables = {'node_sensors': {'method': 'get_node_sensor_readings',
                                 'filter_column': 'nodeID'}}

if __name__ == '__main__':
    parser = mc.get_mc_argument_parser()
    parser.description = """Write M&C records to a CSV file"""
    parser.add_argument('table', help="table to get info from")
    parser.add_argument('--filter_value', help="value to filter on, corresponds to "
                        "'filter_column' the desired table", default=None)
    parser.add_argument('--filename', help="filename to save data to")
    parser.add_argument('--start-date', dest='start_date', help="Start date YYYY/MM/DD", default=None)
    parser.add_argument('--start-time', dest='start_time', help="Start time in HH:MM", default='17:00')
    parser.add_argument('--stop-date', dest='stop_date', help="Stop date YYYY/MM/DD", default=None)
    parser.add_argument('--stop-time', dest='stop_time', help="Stop time in HH:MM", default='7:00')
    parser.add_argument('-l', '--last-period', dest='last_period', default=None,
                        help="Time period from present for data (in minutes).  If present ignores start/stop.")

    args = parser.parse_args()

    variables = cm_utils.listify(args.variables)
    if args.last_period:
        from astropy.time import TimeDelta
        stop_time = Time.now()
        start_time = stop_time - TimeDelta(float(args.last_period) / (60.0 * 24.0), format='jd')
    else:
        start_time = cm_utils.get_astropytime(args.start_date, args.start_time)
        stop_time = cm_utils.get_astropytime(args.stop_date, args.stop_time)

    db = mc.connect_to_mc_db(args)
    session = db.sessionmaker()

    method_kwargs = {'starttime': start_time, 'stoptime': stop_time,
                     valid_tables[args.table][filter_column]: args.filter_value,
                     'write_to_file': True, 'filename': args.filename}
    getattr(session, valid_tables[args.table][method])(**method_kwargs)
<commit_msg>bug fix<commit_after>#! /usr/bin/env python
# -*- mode: python; coding: utf-8 -*-
# Copyright 2018 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""script to write M&C records to a CSV file

"""
from __future__ import absolute_import, division, print_function

import sys
from astropy.time import Time

from hera_mc import mc, cm_utils

valid_tables = {'node_sensors': {'method': 'get_node_sensor_readings',
                                 'filter_column': 'nodeID'}}

if __name__ == '__main__':
    parser = mc.get_mc_argument_parser()
    parser.description = """Write M&C records to a CSV file"""
    parser.add_argument('table', help="table to get info from")
    parser.add_argument('--filter_value', help="value to filter on, corresponds to "
                        "'filter_column' the desired table", default=None)
    parser.add_argument('--filename', help="filename to save data to")
    parser.add_argument('--start-date', dest='start_date', help="Start date YYYY/MM/DD", default=None)
    parser.add_argument('--start-time', dest='start_time', help="Start time in HH:MM", default='17:00')
    parser.add_argument('--stop-date', dest='stop_date', help="Stop date YYYY/MM/DD", default=None)
    parser.add_argument('--stop-time', dest='stop_time', help="Stop time in HH:MM", default='7:00')
    parser.add_argument('-l', '--last-period', dest='last_period', default=None,
                        help="Time period from present for data (in minutes).  If present ignores start/stop.")

    args = parser.parse_args()

    if args.last_period:
        from astropy.time import TimeDelta
        stop_time = Time.now()
        start_time = stop_time - TimeDelta(float(args.last_period) / (60.0 * 24.0), format='jd')
    else:
        start_time = cm_utils.get_astropytime(args.start_date, args.start_time)
        stop_time = cm_utils.get_astropytime(args.stop_date, args.stop_time)

    db = mc.connect_to_mc_db(args)
    session = db.sessionmaker()

    method_kwargs = {'starttime': start_time, 'stoptime': stop_time,
                     valid_tables[args.table][filter_column]: args.filter_value,
                     'write_to_file': True, 'filename': args.filename}
    getattr(session, valid_tables[args.table][method])(**method_kwargs)
<|endoftext|>


<commit_before><commit_msg>minior bug: minmax<commit_after><|endoftext|>


<commit_before>from ceph_medic import check, generate, log
import sys
import os
from tambo import Transport
from execnet.gateway_bootstrap import HostNotFound
import ceph_medic
from ceph_medic.decorators import catches
#from ceph_medic.loader import load_config
from ceph_medic.util import configuration
from ceph_medic import terminal

log.setup()

class Medic(object):
    _help = """
ceph-medic: A utility to run system checks on a Ceph cluster.

Version: {version}

Global Options:
  --ignore              Comma-separated list of errors and warnings to ignore.
  --config              Path to a specific configuration file. Overrides the default:
                        $HOME/.cephdoctor.
  --version, version    Shows the current installed version
  --inventory           Prefer a ceph-ansible inventory (hosts) file instead of default
                        (cwd, /etc/ansible/hosts) locations
  -s, no-capture        Avoids capturing stderr and stdout
  --debug               Doesn't remove internal tracebacks

{sub_help}

{config_path_header}: {config_path}
{hosts_file_header}: {hosts_file}
    """
    mapper = {
        'check': check.Check,
        'generate': generate.Generate,
    }

    def __init__(self, argv=None, parse=True):
        if argv is None:
            argv = sys.argv
        if parse:
            self.main(argv)

    def help(self, sub_help=None):
        if self.hosts_file is None:
            hosts_file_header = terminal.red('Loaded Hosts file')
            hosts_file = 'No hosts file found in cwd, /etc/ansible/, or configured'
        else:
            hosts_file_header = terminal.green('Loaded Hosts file')
            hosts_file = self.hosts_file
        return self._help.format(
            version=ceph_medic.__version__,
            config_path=self.config_path,
            config_path_header=terminal.green('Loaded Config Path'),
            hosts_file=hosts_file,
            hosts_file_header=hosts_file_header,
            sub_help=sub_help
        )

    @catches((RuntimeError, KeyboardInterrupt, HostNotFound))
    def main(self, argv):
        options = ['--ssh-config', '--inventory', '--ignore', '--config', 'no-capture', '-s', '--debug']
        parser = Transport(
            argv, options=options,
            check_help=False,
            check_version=False
        )
        parser.parse_args()
        #default_config_path = os.path.expanduser('~/.cephmedic')
        self.config_path = parser.get('--config', configuration.location())
        #config_path = parser.get('--config')

        # load medic configuration
        loaded_config = configuration.load(path=parser.get('--config', self.config_path))
        # update the module-wide configuration object
        ceph_medic.config.update(configuration.get_overrides(loaded_config))

        # SSH config
        ceph_medic.config['ssh_config'] = parser.get('--ssh-config')

        # Hosts file
        self.hosts_file = parser.get('--inventory', configuration.get_host_file())

        # find the hosts files, by the CLI first, fallback to the configuration
        # file, and lastly if none of those are found or defined, try to load
        # from well known locations (cwd, and /etc/ansible/)
        loaded_hosts = configuration.load_hosts(
            parser.get('--inventory',
                       ceph_medic.config.get('--inventory', self.hosts_file)))

        ceph_medic.config['hosts_file'] = loaded_hosts.filename
        self.hosts_file = loaded_hosts.filename

        parser.catch_version = ceph_medic.__version__
        parser.mapper = self.mapper
        parser.catch_help = self.help(parser.subhelp())
        if len(argv) <= 1:
            return parser.print_help()
        ceph_medic.config['config_path'] = self.config_path
        parser.dispatch()
        parser.catches_help()
        parser.catches_version()
<commit_msg>main: display configured nodes in the help menu when loaded<commit_after>from ceph_medic import check, generate, log
import sys
import os
from textwrap import dedent
from tambo import Transport
from execnet.gateway_bootstrap import HostNotFound
import ceph_medic
from ceph_medic.decorators import catches
#from ceph_medic.loader import load_config
from ceph_medic.util import configuration
from ceph_medic import terminal

log.setup()

class Medic(object):
    _help = """
ceph-medic: A utility to run system checks on a Ceph cluster.

Version: {version}

Global Options:
  --ignore              Comma-separated list of errors and warnings to ignore.
  --config              Path to a specific configuration file. Overrides the default:
                        $HOME/.cephdoctor.
  --version, version    Shows the current installed version
  --inventory           Prefer a ceph-ansible inventory (hosts) file instead of default
                        (cwd, /etc/ansible/hosts) locations
  -s, no-capture        Avoids capturing stderr and stdout
  --debug               Doesn't remove internal tracebacks

{sub_help}

{config_path_header}: {config_path}
{hosts_file_header}: {hosts_file}
{configured_nodes}
    """
    mapper = {
        'check': check.Check,
        'generate': generate.Generate,
    }

    def __init__(self, argv=None, parse=True):
        if argv is None:
            argv = sys.argv
        if parse:
            self.main(argv)

    def help(self, sub_help=None):
        if self.hosts_file is None:
            hosts_file_header = terminal.red('Loaded Hosts file')
            hosts_file = 'No hosts file found in cwd, /etc/ansible/, or configured'
        else:
            hosts_file_header = terminal.green('Loaded Hosts file')
            hosts_file = self.hosts_file
        return self._help.format(
            version=ceph_medic.__version__,
            config_path=self.config_path,
            config_path_header=terminal.green('Loaded Config Path'),
            hosts_file=hosts_file,
            hosts_file_header=hosts_file_header,
            sub_help=sub_help,
            configured_nodes=self.configured_nodes
        )

    @property
    def configured_nodes(self):
        _help = dedent("""
            Configured nodes (loaded from inventory file):
              OSDs: {osd_node_count}
              MONs: {mon_node_count}
              MGRs: {mgr_node_count}
              MDSs: {mds_node_count}
              RGWs: {rgw_node_count}""")
        if self.hosts_file:  # we have nodes that have been loaded
            nodes = ceph_medic.config['nodes']
            return _help.format(
                osd_node_count=len(nodes.get('osds', [])),
                mon_node_count=len(nodes.get('mons', [])),
                mds_node_count=len(nodes.get('mdss', [])),
                mgr_node_count=len(nodes.get('mgrs', [])),
                rgw_node_count=len(nodes.get('rgws', []))
            )
        return ''

    @catches((RuntimeError, KeyboardInterrupt, HostNotFound))
    def main(self, argv):
        options = ['--ssh-config', '--inventory', '--ignore', '--config', 'no-capture', '-s', '--debug']
        parser = Transport(
            argv, options=options,
            check_help=False,
            check_version=False
        )
        parser.parse_args()
        #default_config_path = os.path.expanduser('~/.cephmedic')
        self.config_path = parser.get('--config', configuration.location())
        #config_path = parser.get('--config')

        # load medic configuration
        loaded_config = configuration.load(path=parser.get('--config', self.config_path))
        # update the module-wide configuration object
        ceph_medic.config.update(configuration.get_overrides(loaded_config))

        # SSH config
        ceph_medic.config['ssh_config'] = parser.get('--ssh-config')

        # Hosts file
        self.hosts_file = parser.get('--inventory', configuration.get_host_file())

        # find the hosts files, by the CLI first, fallback to the configuration
        # file, and lastly if none of those are found or defined, try to load
        # from well known locations (cwd, and /etc/ansible/)
        loaded_hosts = configuration.load_hosts(
            parser.get('--inventory',
                       ceph_medic.config.get('--inventory', self.hosts_file)))
        ceph_medic.config['nodes'] = loaded_hosts.nodes
        ceph_medic.config['hosts_file'] = loaded_hosts.filename
        self.hosts_file = loaded_hosts.filename

        parser.catch_version = ceph_medic.__version__
        parser.mapper = self.mapper
        parser.catch_help = self.help(parser.subhelp())
        if len(argv) <= 1:
            return parser.print_help()
        ceph_medic.config['config_path'] = self.config_path
        parser.dispatch()
        parser.catches_help()
        parser.catches_version()
<|endoftext|>


<commit_before>import re

from django.contrib import admin
from django.contrib.auth.models import User
from django import forms

admin.site.unregister(User) # Deregister the built-in User model

import datetime
from hado.models import  *
from hado.forms import *

# Inline classes
class PaymentInline(admin.TabularInline):
	model = Payment
	extra = 1

	fields = ('date_paid', 'amount', 'contract', 'method', 'desc')

	def formfield_for_foreignkey(self, db_field, request, **kwargs):
		if db_field.name == "contract":

			# Assuming we're editing user in the address eg. "/admin/hado/user/3/"
			# So we extract the user_id portion from that path and use it in our queryset filter
			m = re.search("/.+\/(?P<id>\d+)\/?", request.path_info)
			if m is not None:
				user_id = m.groupdict()['id']

			kwargs["queryset"] = Contract.objects.filter(user__id = user_id)
			return db_field.formfield(**kwargs)
		return super(PaymentInline, self).formfield_for_foreignkey(db_field, request, **kwargs)

class TierInline(admin.TabularInline):
	model = Tier

class ContractInline(admin.TabularInline):
	model = Contract
	extra = 0
	inlines = [ PaymentInline, ]

# ModelAdmin classes
class PaymentAdmin(admin.ModelAdmin):
	form = PaymentForm

	fields = ('user', 'contract', 'date_paid', 'amount', 'method', 'desc')

	def formfield_for_foreignkey(self, db_field, request, **kwargs):
		if db_field.name == "contract":
		    kwargs["queryset"] = Contract.objects.exclude(status = "TER")
		    return db_field.formfield(**kwargs)
		return super(PaymentAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

class ContractAdmin(admin.ModelAdmin):
	inlines = [ PaymentInline, ]

class UserAdmin(admin.ModelAdmin):
	list_display = ('__unicode__', 'email')
	list_display_links = ('__unicode__',)
	inlines = [ ContractInline, PaymentInline ]
	fieldsets = (
		(None, {
			'fields': ('username', ('first_name', 'last_name'), 'email')
		}),
	)

admin.site.register(User, UserAdmin)
admin.site.register(ContractType)
admin.site.register(Contract, ContractAdmin)
admin.site.register(Tier)
admin.site.register(Payment, PaymentAdmin)
<commit_msg>Making the PaymentInline use the custom PaymentForm in hado.form, which has added sanity checks<commit_after>import re

from django.contrib import admin
from django.contrib.auth.models import User
from django import forms

admin.site.unregister(User) # Deregister the built-in User model

import datetime
from hado.models import  *
from hado.forms import *

# Inline classes
class PaymentInline(admin.TabularInline):
	model = Payment
	form = PaymentForm
	extra = 1

	fields = ('date_paid', 'amount', 'contract', 'method', 'desc')

	def formfield_for_foreignkey(self, db_field, request, **kwargs):
		if db_field.name == "contract":

			# Assuming we're editing user in the address eg. "/admin/hado/user/3/"
			# So we extract the user_id portion from that path and use it in our queryset filter
			m = re.search("/.+\/(?P<id>\d+)\/?", request.path_info)
			if m is not None:
				user_id = m.groupdict()['id']

			kwargs["queryset"] = Contract.objects.filter(user__id = user_id)
			return db_field.formfield(**kwargs)
		return super(PaymentInline, self).formfield_for_foreignkey(db_field, request, **kwargs)

class TierInline(admin.TabularInline):
	model = Tier

class ContractInline(admin.TabularInline):
	model = Contract
	extra = 0
	inlines = [ PaymentInline, ]

# ModelAdmin classes
class PaymentAdmin(admin.ModelAdmin):
	form = PaymentForm

	fields = ('user', 'contract', 'date_paid', 'amount', 'method', 'desc')

	def formfield_for_foreignkey(self, db_field, request, **kwargs):
		if db_field.name == "contract":
		    kwargs["queryset"] = Contract.objects.exclude(status = "TER")
		    return db_field.formfield(**kwargs)
		return super(PaymentAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

class ContractAdmin(admin.ModelAdmin):
	inlines = [ PaymentInline, ]

class UserAdmin(admin.ModelAdmin):
	list_display = ('__unicode__', 'email')
	list_display_links = ('__unicode__',)
	inlines = [ ContractInline, PaymentInline ]
	fieldsets = (
		(None, {
			'fields': ('username', ('first_name', 'last_name'), 'email')
		}),
	)

admin.site.register(User, UserAdmin)
admin.site.register(ContractType)
admin.site.register(Contract, ContractAdmin)
admin.site.register(Tier)
admin.site.register(Payment, PaymentAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import json
import logging
import os
import requests
from requests_toolbelt import MultipartEncoder
import sys
import time
from twilio import TwilioRestException
from twilio.rest import TwilioRestClient
import yaml
from bottle import route, run, request, abort

# from bt.tn -- index page is triggered by the button itself
#
@route("/", method=['GET', 'POST'])
def from_bttn():
    """
    Processes the press of a bt.tn device

    This function is called from far far away, over the Internet
    """

    print("Button has been pressed")

    try:

        # step 1 - get a room, or create one if needed
        #
        room_id = get_room()

        # step 2 - ensure they are some listeners here
        #
        add_audience(room_id)

        # step 3 - process the action
        #
        process_push(room_id)

        print("Cisco Spark has been updated")
        return "OK\n"

    except Exception as feedback:
        print("ABORTED: fatal error has been encountered")
        print(str(feedback))
        raise
#        return str(feedback)+'\n'

#
# Handle Cisco Spark API
#

def get_room():
    """
    Looks for a suitable Cisco Spark room

    :return: the id of the target room
    :rtype: ``str``

    This function creates a new room if necessary
    """

    print("Looking for Cisco Spark room '{}'".format(settings['room']))

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    response = requests.get(url=url, headers=headers)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    for item in response.json()['items']:
        if settings['room'] in item['title']:
            print("- found it")
            return item['id']

    print("- not found")
    print("Creating Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    payload = {'title': settings['room'] }
    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print("- done")
    return response.json()['id']

def delete_room():
    """
    Deletes the target Cisco Spark room

    This function is useful to restart a clean demo environment
    """

    print("Deleting Cisco Spark room '{}'".format(settings['room']))

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    response = requests.get(url=url, headers=headers)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    actual = False
    for item in response.json()['items']:

        if settings['room'] in item['title']:
            print("- found it")
            print("- DELETING IT")

            url = 'https://api.ciscospark.com/v1/rooms/{}'.format(item['id'])
            headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
            response = requests.delete(url=url, headers=headers)

            if response.status_code != 204:
                raise Exception("Received error code {}".format(response.status_code))

            actual = True

    if actual:
        print("- room will be re-created in Cisco Spark on next button depress")
    else:
        print("- no room with this name yet - it will be created on next button depress")

    settings['shouldAddModerator'] = True

def add_audience(room_id):
    """
    Gives a chance to some listeners to catch updates

    :param room_id: identify the target room
    :type room_id: ``str``

    This function adds pre-defined listeners to a Cisco Spark room if necessary
    """

    if settings['shouldAddModerator'] is False:
        return

    print("Adding moderator to the Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/memberships'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    payload = {'roomId': room_id,
               'personEmail': settings['CISCO_SPARK_BTTN_MAN'],
               'isModerator': 'true' }
    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print("- done")

    settings['shouldAddModerator'] = False

def post_update(room_id, update):
    """
    Updates a Cisco Spark room

    :param room_id: identify the target room
    :type room_id: ``str``

    :param update: content of the update to be posted there
    :type update: ``str`` or ``dict``

    If the update is a simple string, it is sent as such to Cisco Spark.
    Else if it a dictionary, then it is encoded as MIME Multipart.
    """

    print("Posting update to Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/messages'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}

    if isinstance(update, dict):
        update['roomId'] = room_id
        payload = MultipartEncoder(fields=update)
        headers['Content-Type'] = payload.content_type
    else:
        payload = {'roomId': room_id, 'text': update }

    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print('- done, check the room with Cisco Spark client software')

#
# handle Twilio API
#

def call_conference(room_id, numbers):
    """"
    Triggers a conference call

    :param room_id: identify the target room
    :type room_id: ``str``

    :param numbers: a list of phone numbers to call
    :type numbers: ``list``

    This function creates a conference call and adds phone participants
    """
    print("- adding a conference call")
    update = { 'markdown': 'Arranging a conference call'}
    post_update(room_id, update)

    time.sleep(3)

    update = { 'markdown': 'Call [+44 12 34 56 78](tel:+44-12-34-56-78) to join the conference call from any phone.'}
    post_update(room_id, update)

    for line in numbers:
        print("- calling '{}'".format(line))
        update = { 'markdown': 'Adding {} to the conference call'.format(line)}
        post_update(room_id, update)

    time.sleep(5)
    return "Conference call will end when all participants will leave it"

def send_sms(room_id, details):
    """"
    Sends a SMS to target people

    :param room_id: identify the target room
    :type room_id: ``str``

    :param details: what to send and to which numbers
    :type details: ``list``

    This function uses the Tropo API to send a SMS message to target people
    """
    print("- sending a SMS")

    handle = TwilioRestClient(settings['TWILIO_ACCOUNT_SID'],
                              settings['TWILIO_AUTH_TOKEN'])

    message = ''
    from_number = None
    to_numbers = []

    for line in details:
        if not isinstance(line, dict):
            print("- invalid statement: '{}'".format(str(line)))
            update = { 'markdown': 'Unable to send a SMS - check configuration'}
            post_update(room_id, update)
            return

        if line.keys()[0] == 'message':
            message = line['message']

        if line.keys()[0] == 'from':
            from_number = line['from']

        if line.keys()[0] == 'number':
            to_numbers.append(line['number'])

    if len(message) < 4:
        print("- message should have at least 4 characters: '{}'".format(str(message)))
        update = { 'markdown': 'No SMS message to send - check configuration'}
        post_update(room_id, update)
        return

    print("- sending '{}'".format(message))
    update = { 'markdown': "Sending a SMS '{}'".format(message)}
    post_update(room_id, update)

    if len(to_numbers) < 1:
        print("- no phone number has been defined")
        update = { 'markdown': 'No target phone number for SMS - check configuration'}
        post_update(room_id, update)
        return

    if from_number is None:
        from_number = to_numbers[0]

    for number in to_numbers:
        print("- sending to '{}'".format(number))

        try:
            message = handle.messages.create(body=message,
                                             to=number,
                                             from_=from_number)

            update = { 'markdown': 'Sending SMS to {}'.format(number)}
            post_update(room_id, update)

        except TwilioRestException as feedback:
            print("- {}".format(str(feedback)))

            update = { 'markdown': 'Error while sending SMS'}
            post_update(room_id, update)

    time.sleep(5)
    return "SMS has been sent"

#
# behaviour of this software robot
#

def process_push(room_id):
    """
    Processes one push of the button

    :param room_id: identify the target room
    :type room_id: ``str``

    This function monitors the number of button pushes, and uses the appropriate
    action as defined in the configuration file, under the `bt.tn:` keyword.

    The action can be either:

    * send a text message to the room with `text:` statement
    * send a Markdown message to the room with `markdown:` statement
    * upload a file with `file:`, `label:` and `type:`
    * arrange a conference with numbers under the 'conference:' statement
    * a combination of any previous

    """

    print("Building update")

    items = settings['bt.tn']
    if settings['count'] < len(items):
        print("- using item {}".format(settings['count']))
        item = items[ settings['count'] ]

        update = {'text': ''}

        # textual message
        #
        if 'markdown' in item:

            text = 'using markdown content'
            update['markdown'] = item['markdown']

        elif 'message' in item:

            text = "'{}".format(item['message'])
            update['text'] += item['message']+'\n'

        # file upload
        #
        if 'file' in item:

            print("- attaching file {}".format(item['file']))

            if 'label' in item:
                text = item['label']
                update['text'] += "'{}'".format(item['label'])+'\n'

            else:
                text = item['file']

            if 'type' in item:
                type = item['type']
            else:
                type = 'application/octet-stream'

            update['files'] = (text, open(item['file'], 'rb'), type)

        # conference call
        #
        if 'conference' in item:

            update['text'] += call_conference(room_id, item['conference'])+'\n'

        # send a SMS
        #
        if 'sms' in item:

            update['text'] += send_sms(room_id, item['sms'])+'\n'

    else:
        text = 'ping {}'.format(settings['count'])
        update = text

    settings['count'] += 1

    print("- {}".format(text))
    post_update(room_id, update)

def configure(name="settings.yaml"):
    """
    Reads configuration information

    :param name: the file that contains configuration information
    :type name: ``str``

    The function loads configuration from the file and from the environment.
    Port number can be set from the command line.

    Sample configuration file to illustrate capabilities of the program::

        room: "Green Forge"

        CISCO_SPARK_BTTN_BOT: "YWM2OEG4OGItNTQ5YS00MDU2LThkNWEtMJNkODk3ZDZLOGQ0OVGlZWU1NmYtZWyY"
        CISCO_SPARK_BTTN_MAN: "foo.bar@acme.com"

        port: 8080

        bt.tn:

          - markdown: |
                Green Power has been invoked again
                ==================================

                The [green button](https://d2jaw3pqpetn6l.cloudfront.net/app/uploads/2016/05/27125600/product-images-bttn-normal-green-600x600.jpg) has been pressed, so there is a need for urgent action.

                Some context to this event: *Italic*, **bold**, and `monospace`.
                Itemized lists look like this:

                  * this one
                  * that one
                  * the other one

                Unicode is supported. \xe2 And [Incident Management](https://en.wikipedia.org/wiki/Incident_management_(ITSM)) too.
                Call Global Service Center at [+44 12 34 56 78](tel:+44-12-34-56-78) if people are late to join this room.
                We will continue to feed this room with information.

          - file: IncidentReportForm.pdf
            type: "application/pdf"
            label: "Print and fill this report"

          - file: bt.tn.png
            type: "image/png"
            label: "European buttons that rock"

          - file: spark.png
            type: "image/png"
            label: "Cisco Spark brings things and human beings together"

          - file: dimension-data.png
            type: "image/png"
            label: "Build new integrated systems and manage them"

    """

    print("Loading the configuration")

    with open(name, 'r') as stream:
        try:
            settings = yaml.load(stream)
            print("- from '{}'".format(name))
        except Exception as feedback:
            logging.error(str(feedback))
            sys.exit(1)

    if "room" not in settings:
        logging.error("Missing room: configuration information")
        sys.exit(1)

    if "bt.tn" not in settings:
        logging.error("Missing bt.tn: configuration information")
        sys.exit(1)

    if len(sys.argv) > 1:
        try:
            port_number = int(sys.argv[1])
        except:
            logging.error("Invalid port_number specified")
            sys.exit(1)
    elif "port" in settings:
        port_number = int(settings["port"])
    else:
        port_number = 80
    settings['port'] = port_number

    if 'DEBUG' in settings:
        debug = settings['DEBUG']
    else:
        debug = os.environ.get('DEBUG', False)
    settings['DEBUG'] = debug
    if debug:
        logging.basicConfig(level=logging.DEBUG)

    if 'CISCO_SPARK_BTTN_BOT' not in settings:
        token = os.environ.get('CISCO_SPARK_BTTN_BOT')
        if token is None:
            logging.error("Missing CISCO_SPARK_BTTN_BOT in the environment")
            sys.exit(1)
        settings['CISCO_SPARK_BTTN_BOT'] = token

    if 'CISCO_SPARK_BTTN_MAN' not in settings:
        emails = os.environ.get('CISCO_SPARK_BTTN_MAN')
        if emails is None:
            logging.error("Missing CISCO_SPARK_NTTN_MAN in the environment")
            sys.exit(1)
        settings['CISCO_SPARK_BTTN_MAN'] = emails

    if 'TWILIO_ACCOUNT_SID' not in settings:
        token = os.environ.get('TWILIO_ACCOUNT_SID')
        if token is None:
            logging.error("Missing TWILIO_ACCOUNT_SID in the environment")
            sys.exit(1)
        settings['TWILIO_ACCOUNT_SID'] = token

    if 'TWILIO_AUTH_TOKEN' not in settings:
        token = os.environ.get('TWILIO_AUTH_TOKEN')
        if token is None:
            logging.error("Missing TWILIO_AUTH_TOKEN in the environment")
            sys.exit(1)
        settings['TWILIO_AUTH_TOKEN'] = token

    settings['count'] = 0

    return settings

# the program launched from the command line
#
if __name__ == "__main__":

    # read configuration file, look at the environment
    #
    settings = configure()

    # create a clean environment for the demo
    #
    delete_room()

    # wait for button pushes and other web requests
    #
    print("Preparing for web requests")
    run(host='0.0.0.0',
        port=settings['port'],
        debug=settings['DEBUG'],
        server=os.environ.get('SERVER', "auto"))
<commit_msg>streamline SMS handling<commit_after>#!/usr/bin/env python
import json
import logging
import os
import requests
from requests_toolbelt import MultipartEncoder
import sys
import time
from twilio import TwilioRestException
from twilio.rest import TwilioRestClient
import yaml
from bottle import route, run, request, abort

# from bt.tn -- index page is triggered by the button itself
#
@route("/", method=['GET', 'POST'])
def from_bttn():
    """
    Processes the press of a bt.tn device

    This function is called from far far away, over the Internet
    """

    print("Button has been pressed")

    try:

        # step 1 - get a room, or create one if needed
        #
        room_id = get_room()

        # step 2 - ensure they are some listeners here
        #
        add_audience(room_id)

        # step 3 - process the action
        #
        process_push(room_id)

        print("Cisco Spark has been updated")
        return "OK\n"

    except Exception as feedback:
        print("ABORTED: fatal error has been encountered")
        print(str(feedback))
        raise

#
# Handle Cisco Spark API
#

def get_room():
    """
    Looks for a suitable Cisco Spark room

    :return: the id of the target room
    :rtype: ``str``

    This function creates a new room if necessary
    """

    print("Looking for Cisco Spark room '{}'".format(settings['room']))

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    response = requests.get(url=url, headers=headers)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    for item in response.json()['items']:
        if settings['room'] in item['title']:
            print("- found it")
            return item['id']

    print("- not found")
    print("Creating Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    payload = {'title': settings['room'] }
    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print("- done")
    return response.json()['id']

def delete_room():
    """
    Deletes the target Cisco Spark room

    This function is useful to restart a clean demo environment
    """

    print("Deleting Cisco Spark room '{}'".format(settings['room']))

    url = 'https://api.ciscospark.com/v1/rooms'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    response = requests.get(url=url, headers=headers)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    actual = False
    for item in response.json()['items']:

        if settings['room'] in item['title']:
            print("- found it")
            print("- DELETING IT")

            url = 'https://api.ciscospark.com/v1/rooms/{}'.format(item['id'])
            headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
            response = requests.delete(url=url, headers=headers)

            if response.status_code != 204:
                raise Exception("Received error code {}".format(response.status_code))

            actual = True

    if actual:
        print("- room will be re-created in Cisco Spark on next button depress")
    else:
        print("- no room with this name yet - it will be created on next button depress")

    settings['shouldAddModerator'] = True

def add_audience(room_id):
    """
    Gives a chance to some listeners to catch updates

    :param room_id: identify the target room
    :type room_id: ``str``

    This function adds pre-defined listeners to a Cisco Spark room if necessary
    """

    if settings['shouldAddModerator'] is False:
        return

    print("Adding moderator to the Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/memberships'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}
    payload = {'roomId': room_id,
               'personEmail': settings['CISCO_SPARK_BTTN_MAN'],
               'isModerator': 'true' }
    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print("- done")

    settings['shouldAddModerator'] = False

def post_update(room_id, update):
    """
    Updates a Cisco Spark room

    :param room_id: identify the target room
    :type room_id: ``str``

    :param update: content of the update to be posted there
    :type update: ``str`` or ``dict``

    If the update is a simple string, it is sent as such to Cisco Spark.
    Else if it a dictionary, then it is encoded as MIME Multipart.
    """

    print("Posting update to Cisco Spark room")

    url = 'https://api.ciscospark.com/v1/messages'
    headers = {'Authorization': 'Bearer '+settings['CISCO_SPARK_BTTN_BOT']}

    if isinstance(update, dict):
        update['roomId'] = room_id
        payload = MultipartEncoder(fields=update)
        headers['Content-Type'] = payload.content_type
    else:
        payload = {'roomId': room_id, 'text': update }

    response = requests.post(url=url, headers=headers, data=payload)

    if response.status_code != 200:
        print(response.json())
        raise Exception("Received error code {}".format(response.status_code))

    print('- done, check the room with Cisco Spark client software')

#
# handle Twilio API
#

def call_conference(room_id, numbers):
    """"
    Triggers a conference call

    :param room_id: identify the target room
    :type room_id: ``str``

    :param numbers: a list of phone numbers to call
    :type numbers: ``list``

    This function creates a conference call and adds phone participants
    """
    print("- adding a conference call")
    update = { 'markdown': 'Arranging a conference call'}
    post_update(room_id, update)

    time.sleep(3)

    update = { 'markdown': 'Call [+44 12 34 56 78](tel:+44-12-34-56-78) to join the conference call from any phone.'}
    post_update(room_id, update)

    for line in numbers:
        print("- calling '{}'".format(line))
        update = { 'markdown': 'Adding {} to the conference call'.format(line)}
        post_update(room_id, update)

    time.sleep(5)
    return "Conference call will end when all participants will leave it"

def send_sms(room_id, details):
    """"
    Sends a SMS to target people

    :param room_id: identify the target room
    :type room_id: ``str``

    :param details: what to send and to which numbers
    :type details: ``list``

    This function uses the Twilio API to send a SMS message to target people
    """
    print("- sending a SMS")

    handle = TwilioRestClient(settings['TWILIO_ACCOUNT_SID'],
                              settings['TWILIO_AUTH_TOKEN'])

    message = ''
    from_number = None
    to_numbers = []

    for line in details:
        if not isinstance(line, dict):
            print("- invalid statement: '{}'".format(str(line)))
            update = { 'markdown': 'Unable to send a SMS - check configuration'}
            post_update(room_id, update)
            return

        if line.keys()[0] == 'message':
            message = line['message']

        if line.keys()[0] == 'from':
            from_number = line['from']

        if line.keys()[0] == 'number':
            to_numbers.append(line['number'])

    if len(message) < 4:
        print("- message should have at least 4 characters: '{}'".format(str(message)))
        update = { 'markdown': 'No SMS message to send - check configuration'}
        post_update(room_id, update)
        return

    print("- sending '{}'".format(message))

    if len(to_numbers) < 1:
        print("- no phone number has been defined")
        update = { 'markdown': 'No target phone number for SMS - check configuration'}
        post_update(room_id, update)
        return

    if from_number is None:
        from_number = to_numbers[0]

    for number in to_numbers:
        print("- sending to '{}'".format(number))

        try:
            handle.messages.create(body=message,
                                   to=number,
                                   from_=from_number)

        except TwilioRestException as feedback:
            print("- {}".format(str(feedback)))
            return

    update = { 'markdown': "SMS '{}' has been sent to '{}'".format(message,
                                                   ', '.join(to_numbers))}
    post_update(room_id, update)

#
# behaviour of this software robot
#

def process_push(room_id):
    """
    Processes one push of the button

    :param room_id: identify the target room
    :type room_id: ``str``

    This function monitors the number of button pushes, and uses the appropriate
    action as defined in the configuration file, under the `bt.tn:` keyword.

    The action can be either:

    * send a text message to the room with `text:` statement
    * send a Markdown message to the room with `markdown:` statement
    * upload a file with `file:`, `label:` and `type:`
    * arrange a conference with numbers under the 'conference:' statement
    * a combination of any previous

    """

    print("Building update")

    items = settings['bt.tn']
    if settings['count'] < len(items):
        print("- using item {}".format(settings['count']))
        item = items[ settings['count'] ]

        update = {'text': ''}

        # textual message
        #
        if 'markdown' in item:

            text = 'using markdown content'
            update['markdown'] = item['markdown']

        elif 'message' in item:

            text = "'{}".format(item['message'])
            update['text'] += item['message']+'\n'

        # file upload
        #
        if 'file' in item:

            print("- attaching file {}".format(item['file']))

            if 'label' in item:
                text = item['label']
                update['text'] += "'{}'".format(item['label'])+'\n'

            else:
                text = item['file']

            if 'type' in item:
                type = item['type']
            else:
                type = 'application/octet-stream'

            update['files'] = (text, open(item['file'], 'rb'), type)

        # conference call
        #
        if 'conference' in item:
            call_conference(room_id, item['conference'])

        # send a SMS
        #
        if 'sms' in item:
            send_sms(room_id, item['sms'])

    else:
        text = 'ping {}'.format(settings['count'])
        update = text

    settings['count'] += 1

    print("- {}".format(text))
    post_update(room_id, update)

def configure(name="settings.yaml"):
    """
    Reads configuration information

    :param name: the file that contains configuration information
    :type name: ``str``

    The function loads configuration from the file and from the environment.
    Port number can be set from the command line.

    Sample configuration file to illustrate capabilities of the program::

        room: "Green Forge"

        CISCO_SPARK_BTTN_BOT: "YWM2OEG4OGItNTQ5YS00MDU2LThkNWEtMJNkODk3ZDZLOGQ0OVGlZWU1NmYtZWyY"
        CISCO_SPARK_BTTN_MAN: "foo.bar@acme.com"

        port: 8080

        bt.tn:

          - markdown: |
                Green Power has been invoked again
                ==================================

                The [green button](https://d2jaw3pqpetn6l.cloudfront.net/app/uploads/2016/05/27125600/product-images-bttn-normal-green-600x600.jpg) has been pressed, so there is a need for urgent action.

                Some context to this event: *Italic*, **bold**, and `monospace`.
                Itemized lists look like this:

                  * this one
                  * that one
                  * the other one

                Unicode is supported. \xe2 And [Incident Management](https://en.wikipedia.org/wiki/Incident_management_(ITSM)) too.
                Call Global Service Center at [+44 12 34 56 78](tel:+44-12-34-56-78) if people are late to join this room.
                We will continue to feed this room with information.

          - file: IncidentReportForm.pdf
            type: "application/pdf"
            label: "Print and fill this report"

          - file: bt.tn.png
            type: "image/png"
            label: "European buttons that rock"

          - file: spark.png
            type: "image/png"
            label: "Cisco Spark brings things and human beings together"

          - file: dimension-data.png
            type: "image/png"
            label: "Build new integrated systems and manage them"

    """

    print("Loading the configuration")

    with open(name, 'r') as stream:
        try:
            settings = yaml.load(stream)
            print("- from '{}'".format(name))
        except Exception as feedback:
            logging.error(str(feedback))
            sys.exit(1)

    if "room" not in settings:
        logging.error("Missing room: configuration information")
        sys.exit(1)

    if "bt.tn" not in settings:
        logging.error("Missing bt.tn: configuration information")
        sys.exit(1)

    if len(sys.argv) > 1:
        try:
            port_number = int(sys.argv[1])
        except:
            logging.error("Invalid port_number specified")
            sys.exit(1)
    elif "port" in settings:
        port_number = int(settings["port"])
    else:
        port_number = 80
    settings['port'] = port_number

    if 'DEBUG' in settings:
        debug = settings['DEBUG']
    else:
        debug = os.environ.get('DEBUG', False)
    settings['DEBUG'] = debug
    if debug:
        logging.basicConfig(level=logging.DEBUG)

    if 'CISCO_SPARK_BTTN_BOT' not in settings:
        token = os.environ.get('CISCO_SPARK_BTTN_BOT')
        if token is None:
            logging.error("Missing CISCO_SPARK_BTTN_BOT in the environment")
            sys.exit(1)
        settings['CISCO_SPARK_BTTN_BOT'] = token

    if 'CISCO_SPARK_BTTN_MAN' not in settings:
        emails = os.environ.get('CISCO_SPARK_BTTN_MAN')
        if emails is None:
            logging.error("Missing CISCO_SPARK_NTTN_MAN in the environment")
            sys.exit(1)
        settings['CISCO_SPARK_BTTN_MAN'] = emails

    if 'TWILIO_ACCOUNT_SID' not in settings:
        token = os.environ.get('TWILIO_ACCOUNT_SID')
        if token is None:
            logging.error("Missing TWILIO_ACCOUNT_SID in the environment")
            sys.exit(1)
        settings['TWILIO_ACCOUNT_SID'] = token

    if 'TWILIO_AUTH_TOKEN' not in settings:
        token = os.environ.get('TWILIO_AUTH_TOKEN')
        if token is None:
            logging.error("Missing TWILIO_AUTH_TOKEN in the environment")
            sys.exit(1)
        settings['TWILIO_AUTH_TOKEN'] = token

    settings['count'] = 0

    return settings

# the program launched from the command line
#
if __name__ == "__main__":

    # read configuration file, look at the environment
    #
    settings = configure()

    # create a clean environment for the demo
    #
    delete_room()

    # wait for button pushes and other web requests
    #
    print("Preparing for web requests")
    run(host='0.0.0.0',
        port=settings['port'],
        debug=settings['DEBUG'],
        server=os.environ.get('SERVER', "auto"))
<|endoftext|>


<commit_before>import datetime

from django.conf import settings
from django.utils.translation import ugettext_lazy as _

import mongoengine

GENDER_CHOICES = (
    ('male', _('Male')),
    ('female', _('Female'))
)

def get_initial_language(request=None):
    return settings.LANGUAGE_CODE

class LanguageField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 5)
        kwargs.setdefault('choices', settings.LANGUAGES)
        kwargs.setdefault('default', get_initial_language)

        super(LanguageField, self).__init__(*args, **kwargs)

class GenderField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 6)
        kwargs.setdefault('choices', GENDER_CHOICES)
        kwargs.setdefault('default', GENDER_CHOICES[1][0])

        super(GenderField, self).__init__(*args, **kwargs)

class LimitedDateTimeField(mongoengine.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_lmit = lower_limit
        super(LimitedDateTimeField, self).__init__(*args, **kwargs)

    def validate(self, value):
        super(LimitedDateTimeField, self).validate(value)

        # TODO: This is not needed right?
        if self.upper_limit and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid limit.')
        if self.lower_limit and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid limit.')

        if isinstance(value, datetime.datetime):
            futureDate = value > self.upper_limit
            futureDate |= value < self.lower_limit
        elif isinstance(value, datetime.date):
            futureDate = value > self.upper_limit.date()
            futureDate |= value < self.lower_limit.date()
        if futureDate:
            self.error(u'Value is out of bonds.')<commit_msg>Fixed typo.<commit_after>import datetime

from django.conf import settings
from django.utils.translation import ugettext_lazy as _

import mongoengine

GENDER_CHOICES = (
    ('male', _('Male')),
    ('female', _('Female'))
)

def get_initial_language(request=None):
    return settings.LANGUAGE_CODE

class LanguageField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 5)
        kwargs.setdefault('choices', settings.LANGUAGES)
        kwargs.setdefault('default', get_initial_language)

        super(LanguageField, self).__init__(*args, **kwargs)

class GenderField(mongoengine.StringField):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('max_length', 6)
        kwargs.setdefault('choices', GENDER_CHOICES)
        kwargs.setdefault('default', GENDER_CHOICES[1][0])

        super(GenderField, self).__init__(*args, **kwargs)

class LimitedDateTimeField(mongoengine.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_lmit = lower_limit
        super(LimitedDateTimeField, self).__init__(*args, **kwargs)

    def validate(self, value):
        super(LimitedDateTimeField, self).validate(value)

        # TODO: This is not needed right?
        if self.upper_limit and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid limit.')
        if self.lower_limit and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid limit.')

        if isinstance(value, datetime.datetime):
            futureDate = value > self.upper_limit
            futureDate |= value < self.lower_limit
        elif isinstance(value, datetime.date):
            futureDate = value > self.upper_limit.date()
            futureDate |= value < self.lower_limit.date()
        if futureDate:
            self.error(u'Value is out of bounds.')<|endoftext|>