<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from openquake.baselib import sap, datastore

def make_figure(plots):
    """
    :param plots: list of pairs (task_name, memory array)
    """
    # NB: matplotlib is imported inside since it is a costly import
    import matplotlib.pyplot as plt

    fig, ax = plt.subplots()
    ax.grid(True)
    ax.set_ylabel('tasks')
    ax.set_ylabel('GB')
    start = 0
    for task_name, mem in plots:
        ax.plot(range(start, start + len(mem)), mem, label=task_name)
        start += len(mem)
    ax.legend()
    return plt

@sap.Script
def plot_memory(calc_id=-1):
    """
    Plot the memory occupation
    """
    dstore = datastore.read(calc_id)
    plots = []
    for task_name in dstore['task_info']:
        mem = dstore['task_info/' + task_name]['mem_gb']
        plots.append((task_name, mem))
    plt = make_figure(plots)
    plt.show()

plot_memory.arg('calc_id', 'calculation ID', type=int)
<commit_msg>Small fix [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from openquake.baselib import sap, datastore

def make_figure(plots):
    """
    :param plots: list of pairs (task_name, memory array)
    """
    # NB: matplotlib is imported inside since it is a costly import
    import matplotlib.pyplot as plt

    fig, ax = plt.subplots()
    ax.grid(True)
    ax.set_xlabel('tasks')
    ax.set_ylabel('GB')
    start = 0
    for task_name, mem in plots:
        ax.plot(range(start, start + len(mem)), mem, label=task_name)
        start += len(mem)
    ax.legend()
    return plt

@sap.Script
def plot_memory(calc_id=-1):
    """
    Plot the memory occupation
    """
    dstore = datastore.read(calc_id)
    plots = []
    for task_name in dstore['task_info']:
        mem = dstore['task_info/' + task_name]['mem_gb']
        plots.append((task_name, mem))
    plt = make_figure(plots)
    plt.show()

plot_memory.arg('calc_id', 'calculation ID', type=int)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Reading risk models for risk calculators
"""
import re
    from urllib import quote_plus
except ImportError:  # Python 3
    from urllib.parse import quote_plus
import collections

import numpy

from openquake.risklib import valid
from openquake.commonlib.node import LiteralNode
from openquake.commonlib import nrml
from openquake.commonlib.sourcewriter import obj_to_node

F64 = numpy.float64

COST_TYPE_REGEX = '|'.join(valid.cost_type.choices)

LOSS_TYPE_KEY = re.compile(
    '(%s|occupants|fragility)_([\w_]+)' % COST_TYPE_REGEX)

def get_risk_files(inputs):
    """
    :param inputs: a dictionary key -> path name
    :returns: a pair (file_type, {cost_type: path})
    """
    vfs = {}
    names = set()
    for key in inputs:
        if key == 'fragility':
            # backward compatibily for .ini files with key fragility_file
            # instead of structural_fragility_file
            vfs['structural'] = inputs['structural_fragility'] = inputs[key]
            names.add('fragility')
            del inputs['fragility']
            continue
        match = LOSS_TYPE_KEY.match(key)
        if match and 'retrofitted' not in key and 'consequence' not in key:
            vfs[match.group(1)] = inputs[key]
            names.add(match.group(2))
    if not names:
        return None, {}
    elif len(names) > 1:
        raise ValueError('Found inconsistent keys %s in the .ini file'
                         % ', '.join(names))
    return names.pop(), vfs

# ########################### vulnerability ############################## #

def filter_vset(elem):
    return elem.tag.endswith('discreteVulnerabilitySet')

@obj_to_node.add('VulnerabilityFunction')
def build_vf_node(vf):
    """
    Convert a VulnerabilityFunction object into a LiteralNode suitable
    for XML conversion.
    """
    nodes = [LiteralNode('imls', {'imt': vf.imt}, vf.imls),
             LiteralNode('meanLRs', {}, vf.mean_loss_ratios),
             LiteralNode('covLRs', {}, vf.covs)]
    return LiteralNode(
        'vulnerabilityFunction',
        {'id': vf.id, 'dist': vf.distribution_name}, nodes=nodes)

def get_risk_models(oqparam, kind=None):
    """
    :param oqparam:
        an OqParam instance
    :param kind:
        vulnerability|vulnerability_retrofitted|fragility|consequence;
        if None it is extracted from the oqparam.file_type attribute
    :returns:
        a dictionary taxonomy -> loss_type -> function
    """
    kind = kind or oqparam.file_type
    rmodels = {}
    for key in oqparam.inputs:
        mo = re.match('(occupants|%s)_%s$' % (COST_TYPE_REGEX, kind), key)
        if mo:
            key_type = mo.group(1)  # the cost_type in the key
            # can be occupants, structural, nonstructural, ...
            rmodel = nrml.parse(oqparam.inputs[key])
            rmodels[key_type] = rmodel
            if rmodel.lossCategory is None:  # NRML 0.4
                continue
            cost_type = str(rmodel.lossCategory)
            rmodel_kind = rmodel.__class__.__name__
            kind_ = kind.replace('_retrofitted', '')  # strip retrofitted
            if not rmodel_kind.lower().startswith(kind_):
                raise ValueError(
                    'Error in the file "%s_file=%s": is '
                    'of kind %s, expected %s' % (
                        key, oqparam.inputs[key], rmodel_kind,
                        kind.capitalize() + 'Model'))
            if cost_type != key_type:
                raise ValueError(
                    'Error in the file "%s_file=%s": lossCategory is of type '
                    '"%s", expected "%s"' % (key, oqparam.inputs[key],
                                             rmodel.lossCategory, key_type))
    rdict = collections.defaultdict(dict)
    if kind == 'fragility':
        limit_states = []
        for loss_type, fm in sorted(rmodels.items()):
            # build a copy of the FragilityModel with different IM levels
            newfm = fm.build(oqparam.continuous_fragility_discretization,
                             oqparam.steps_per_interval)
            for (imt, taxo), ffl in newfm.items():
                if not limit_states:
                    limit_states.extend(fm.limitStates)
                # we are rejecting the case of loss types with different
                # limit states; this may change in the future
                assert limit_states == fm.limitStates, (
                    limit_states, fm.limitStates)
                rdict[taxo][loss_type] = ffl
                # TODO: see if it is possible to remove the attribute
                # below, used in classical_damage
                ffl.steps_per_interval = oqparam.steps_per_interval
        oqparam.limit_states = limit_states
    elif kind == 'consequence':
        rdict = rmodels
    else:  # vulnerability
        cl_risk = oqparam.calculation_mode in ('classical', 'classical_risk')
        # only for classical_risk reduce the loss_ratios
        # to make sure they are strictly increasing
        for loss_type, rm in rmodels.items():
            for (imt, taxo), rf in rm.items():
                rdict[taxo][loss_type] = (
                    rf.strictly_increasing() if cl_risk else rf)
    return rdict
<commit_msg>Fixed a merge error<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Reading risk models for risk calculators
"""
import re
import collections

import numpy

from openquake.risklib import valid
from openquake.commonlib.node import LiteralNode
from openquake.commonlib import nrml
from openquake.commonlib.sourcewriter import obj_to_node

F64 = numpy.float64

COST_TYPE_REGEX = '|'.join(valid.cost_type.choices)

LOSS_TYPE_KEY = re.compile(
    '(%s|occupants|fragility)_([\w_]+)' % COST_TYPE_REGEX)

def get_risk_files(inputs):
    """
    :param inputs: a dictionary key -> path name
    :returns: a pair (file_type, {cost_type: path})
    """
    vfs = {}
    names = set()
    for key in inputs:
        if key == 'fragility':
            # backward compatibily for .ini files with key fragility_file
            # instead of structural_fragility_file
            vfs['structural'] = inputs['structural_fragility'] = inputs[key]
            names.add('fragility')
            del inputs['fragility']
            continue
        match = LOSS_TYPE_KEY.match(key)
        if match and 'retrofitted' not in key and 'consequence' not in key:
            vfs[match.group(1)] = inputs[key]
            names.add(match.group(2))
    if not names:
        return None, {}
    elif len(names) > 1:
        raise ValueError('Found inconsistent keys %s in the .ini file'
                         % ', '.join(names))
    return names.pop(), vfs

# ########################### vulnerability ############################## #

def filter_vset(elem):
    return elem.tag.endswith('discreteVulnerabilitySet')

@obj_to_node.add('VulnerabilityFunction')
def build_vf_node(vf):
    """
    Convert a VulnerabilityFunction object into a LiteralNode suitable
    for XML conversion.
    """
    nodes = [LiteralNode('imls', {'imt': vf.imt}, vf.imls),
             LiteralNode('meanLRs', {}, vf.mean_loss_ratios),
             LiteralNode('covLRs', {}, vf.covs)]
    return LiteralNode(
        'vulnerabilityFunction',
        {'id': vf.id, 'dist': vf.distribution_name}, nodes=nodes)

def get_risk_models(oqparam, kind=None):
    """
    :param oqparam:
        an OqParam instance
    :param kind:
        vulnerability|vulnerability_retrofitted|fragility|consequence;
        if None it is extracted from the oqparam.file_type attribute
    :returns:
        a dictionary taxonomy -> loss_type -> function
    """
    kind = kind or oqparam.file_type
    rmodels = {}
    for key in oqparam.inputs:
        mo = re.match('(occupants|%s)_%s$' % (COST_TYPE_REGEX, kind), key)
        if mo:
            key_type = mo.group(1)  # the cost_type in the key
            # can be occupants, structural, nonstructural, ...
            rmodel = nrml.parse(oqparam.inputs[key])
            rmodels[key_type] = rmodel
            if rmodel.lossCategory is None:  # NRML 0.4
                continue
            cost_type = str(rmodel.lossCategory)
            rmodel_kind = rmodel.__class__.__name__
            kind_ = kind.replace('_retrofitted', '')  # strip retrofitted
            if not rmodel_kind.lower().startswith(kind_):
                raise ValueError(
                    'Error in the file "%s_file=%s": is '
                    'of kind %s, expected %s' % (
                        key, oqparam.inputs[key], rmodel_kind,
                        kind.capitalize() + 'Model'))
            if cost_type != key_type:
                raise ValueError(
                    'Error in the file "%s_file=%s": lossCategory is of type '
                    '"%s", expected "%s"' % (key, oqparam.inputs[key],
                                             rmodel.lossCategory, key_type))
    rdict = collections.defaultdict(dict)
    if kind == 'fragility':
        limit_states = []
        for loss_type, fm in sorted(rmodels.items()):
            # build a copy of the FragilityModel with different IM levels
            newfm = fm.build(oqparam.continuous_fragility_discretization,
                             oqparam.steps_per_interval)
            for (imt, taxo), ffl in newfm.items():
                if not limit_states:
                    limit_states.extend(fm.limitStates)
                # we are rejecting the case of loss types with different
                # limit states; this may change in the future
                assert limit_states == fm.limitStates, (
                    limit_states, fm.limitStates)
                rdict[taxo][loss_type] = ffl
                # TODO: see if it is possible to remove the attribute
                # below, used in classical_damage
                ffl.steps_per_interval = oqparam.steps_per_interval
        oqparam.limit_states = limit_states
    elif kind == 'consequence':
        rdict = rmodels
    else:  # vulnerability
        cl_risk = oqparam.calculation_mode in ('classical', 'classical_risk')
        # only for classical_risk reduce the loss_ratios
        # to make sure they are strictly increasing
        for loss_type, rm in rmodels.items():
            for (imt, taxo), rf in rm.items():
                rdict[taxo][loss_type] = (
                    rf.strictly_increasing() if cl_risk else rf)
    return rdict
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2020, GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import abc
import inspect

class SecondaryPeril(metaclass=abc.ABCMeta):
    @classmethod
    def instances(cls, oqparam):
        inst = []
        for clsname in oqparam.secondary_perils:
            c = globals()[clsname]
            lst = []
            for param in inspect.signature(c).parameters:
                lst.append(getattr(oqparam, param))
            inst.append(c(*lst))
        return inst

    outputs = abc.abstractproperty()

    @abc.abstractmethod
    def prepare(self, sites):
        """Add attributes to sites"""

    @abc.abstractmethod
    def compute(self, mag, gmfs, sites):
        # gmv is an array with (N, M) elements
        return gmfs[:, 0] * .1,  # fake formula

    def __repr__(self):
        return '<%s %s>' % self.__class__.__name__

class FakePeril(SecondaryPeril):
    outputs = ['fake']

    def prepare(self, sites):
        pass

    def compute(self, mag, gmfs, sites):
        # gmv is an array with (N, M) elements
        return gmfs[:, 0] * .1,  # fake formula

class NewarkDisplacement(SecondaryPeril):
    outputs = ['newark_disp', 'prob_disp']

    def __init__(self, c1=-2.71, c2=2.335, c3=-1.478, c4=0.424,
                 crit_accel_threshold=0.05):
        self.c1 = c1
        self.c2 = c2
        self.c3 = c3
        self.c4 = c4
        self.crit_accel_threshold = crit_accel_threshold

    def prepare(self, sites):
        sites.add_col('Fs', float, static_factor_of_safety(
            slope=sites.slope,
            cohesion=sites.cohesion_mid,
            friction_angle=sites.friction_mid,
            saturation_coeff=sites.saturation,
            soil_dry_density=sites.dry_density))
        sites.add_col('crit_accel', float,
                      newmark_critical_accel(sites.Fs, sites.slope))

    def compute(self, mag, gmfs, sites):
        nd = newmark_displ_from_pga_M(
            gmfs[:, 0], sites.critical_accel, mag,
            self.c1, self.c2, self.c3, self.c4, self.crit_accel_threshold)
        return nd, prob_failure_given_displacement(nd)

class HazusLiquefaction(SecondaryPeril):
    outputs = ['liq_prob']

    def __init__(self, map_proportion_flag):
        self.map_proportion_flag = map_proportion_flag

    def prepare(self, sites):
        pass

    def compute(self, mag, gmfs, sites):
        return hazus_liquefaction_probability(
            pga=gmfs[:, 0], mag=mag,
            liq_susc_cat=sites.liq_susc_cat,
            groundwater_depth=sites.gwd,
            do_map_proportion_correction=self.map_proportion_flag),

supported = [cls.__name__ for cls in SecondaryPeril.__subclasses__()]
<commit_msg>Added a docstring [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2020, GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
"""
Subclasses of SecondaryPeril have:

1. a ``__init__`` method with parameters in the job.ini
2. a ``prepare(sitecol)`` method modifying on the site collection, called
   in the ``pre_execute`` phase, i.e. before running the calculation
3. a ``compute(mag, gmfs, sites)`` method called during the calculation of
   the GMFs; gmvs is an array of shape (N1, M) and sites is a (filtered)
   site collection with N1 sites (normally N1 < N, the total number of sites)
4. an ``outputs`` attribute which is a list of column names which will be
   added to the gmf_data array generated by the ground motion calculator

The ``compute`` method will return a tuple with ``O`` arrays where ``O``
is the number of outputs, i.e. one array per each output column.
"""
import abc
import inspect

class SecondaryPeril(metaclass=abc.ABCMeta):
    @classmethod
    def instances(cls, oqparam):
        inst = []
        for clsname in oqparam.secondary_perils:
            c = globals()[clsname]
            lst = []
            for param in inspect.signature(c).parameters:
                lst.append(getattr(oqparam, param))
            inst.append(c(*lst))
        return inst

    outputs = abc.abstractproperty()

    @abc.abstractmethod
    def prepare(self, sites):
        """Add attributes to sites"""

    @abc.abstractmethod
    def compute(self, mag, gmfs, sites):
        # gmv is an array with (N, M) elements
        return gmfs[:, 0] * .1,  # fake formula

    def __repr__(self):
        return '<%s %s>' % self.__class__.__name__

class FakePeril(SecondaryPeril):
    outputs = ['fake']

    def prepare(self, sites):
        pass

    def compute(self, mag, gmfs, sites):
        # gmv is an array with (N, M) elements
        return gmfs[:, 0] * .1,  # fake formula

class NewarkDisplacement(SecondaryPeril):
    outputs = ['newark_disp', 'prob_disp']

    def __init__(self, c1=-2.71, c2=2.335, c3=-1.478, c4=0.424,
                 crit_accel_threshold=0.05):
        self.c1 = c1
        self.c2 = c2
        self.c3 = c3
        self.c4 = c4
        self.crit_accel_threshold = crit_accel_threshold

    def prepare(self, sites):
        sites.add_col('Fs', float, static_factor_of_safety(
            slope=sites.slope,
            cohesion=sites.cohesion_mid,
            friction_angle=sites.friction_mid,
            saturation_coeff=sites.saturation,
            soil_dry_density=sites.dry_density))
        sites.add_col('crit_accel', float,
                      newmark_critical_accel(sites.Fs, sites.slope))

    def compute(self, mag, gmfs, sites):
        nd = newmark_displ_from_pga_M(
            gmfs[:, 0], sites.critical_accel, mag,
            self.c1, self.c2, self.c3, self.c4, self.crit_accel_threshold)
        return nd, prob_failure_given_displacement(nd)

class HazusLiquefaction(SecondaryPeril):
    outputs = ['liq_prob']

    def __init__(self, map_proportion_flag):
        self.map_proportion_flag = map_proportion_flag

    def prepare(self, sites):
        pass

    def compute(self, mag, gmfs, sites):
        return hazus_liquefaction_probability(
            pga=gmfs[:, 0], mag=mag,
            liq_susc_cat=sites.liq_susc_cat,
            groundwater_depth=sites.gwd,
            do_map_proportion_correction=self.map_proportion_flag),

supported = [cls.__name__ for cls in SecondaryPeril.__subclasses__()]
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging
import sys
from datetime import datetime

import pytz
from django.contrib.auth.models import Group, Permission
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist
from django.core.management import BaseCommand

from api.base.celery import app
from osf.management.commands.migratedata import register_nonexistent_models_with_modm, get_modm_model
from osf.management.commands.migraterelations import build_toku_django_lookup_table_cache, format_lookup_key
from osf.models import BlackListGuid, NotificationSubscription
from osf.models import EmbargoTerminationApproval
from osf.models import OSFUser
from osf.models import Tag
from osf.utils.order_apps import get_ordered_models
from website.app import init_app
from website.files.models import StoredFileNode
from .migratedata import set_backend
logger = logging.getLogger('migrations')

class NotGonnaDoItException(Exception):
    pass

# NodeLicenseRecord appears pretty broke
# StoredFileNode has no attribute deleted_by, probably deleted_on too

class Command(BaseCommand):
    help = 'Validates migrations from tokumx to postgres'

    def handle(self, *args, **options):
        django_models = get_ordered_models()

        for django_model in django_models:

            if not hasattr(django_model, 'modm_model_path'):
                logger.info('################################################\n'
                      '{} doesn\'t have a modm_model_path\n'
                      '################################################'.format(
                    django_model._meta.model.__name__))
                continue
            elif django_model.__subclasses__() != []:
                logger.info('Skipping {}.{}'.format(django_model.__module__, django_model))
                continue
            elif django_model is Tag or django_model is BlackListGuid:
                # TODO we'll do this by hand because tags are special
                # TODO we'll do blacklistguids by hand because they've got a bunch of new ones that don't exist in modm
                # TODO specifically, from register_none_existent_models
                continue
            do_model.delay(django_model)

@app.task()
def do_model(django_model):
    set_backend()
    init_app(routes=False, attach_request_handlers=False, fixtures=False)
    register_nonexistent_models_with_modm()

    page_size = django_model.migration_page_size

    validate_model_data(django_model, page_size=page_size)

def validate_m2m_field(field_name, django_obj, modm_obj):
    field = getattr(django_obj, field_name)

    # handle both forward and reverse rels
    if hasattr(django_obj, field_name):
        manager = field
        # skip groups and permissions
        if manager.model is Group or manager.model is Permission:
            return
        primary_identifier_name = manager.model.primary_identifier_name
    else:
        manager_name = '{}_set'.format(field.through._meta.object_name).lower()
        manager = getattr(django_obj, manager_name)
        try:
            primary_identifier_name = field.through._meta.model.primary_identifier_name
        except AttributeError:
            primary_identifier_name = '_id'
    django_guids = manager.all().values_list(primary_identifier_name, flat=True)
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    modm_guids = [obj._id for obj in getattr(modm_obj, modm_field_name)]
    for django_guid in django_guids:
        try:
            assert django_guid in modm_guids, '{} for model {}.{} was not in modm guids for field {}'.format(django_guid, django_obj._meta.model.__module__, django_obj._meta.model.__name__, modm_field_name)
        except AssertionError as ex:
            logger.error(ex)

def validate_fk_relation(field_name, django_obj, modm_obj):
    if field_name in ['content_type', 'content_type_pk', 'content_type_id']:
        # modm doesn't have gfk
        return

    if isinstance(modm_obj, StoredFileNode) and field_name == 'deleted_by':
        return

    if django_obj._meta.model is EmbargoTerminationApproval and field_name == 'initiated_by':
        # EmbargoTerminationApproval didn't have an initiated_by in modm even though it was supposed to.
        return

    django_field_value = getattr(django_obj, field_name)
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    if django_obj._meta.model is NotificationSubscription and field_name in ['user', 'node']:
        modm_field_name = 'owner'

    modm_field_value = getattr(modm_obj, modm_field_name)
    if modm_field_value and django_field_value:
        try:
            assert modm_field_value._id == django_field_value._id, 'Modm field {} of obj {}:{} with value of {} doesn\'t equal django field with value {}'.format(field_name, type(modm_obj), modm_obj._id, modm_field_value._id, django_field_value._id)
        except AssertionError as ex:
            logger.error(ex)
    elif modm_field_value is not None and django_field_value is None:
        logger.info('{} of {!r} was None in django but {} in modm'.format(field_name, modm_obj, django_obj, modm_field_value))

def validate_basic_field(field_name, django_obj, modm_obj):
    if field_name in ['id', 'pk', 'object_id', 'guid_string', 'last_login', 'is_superuser']:
        # modm doesn't have these
        return
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    if modm_field_name is False:
        return

    modm_value = getattr(modm_obj, modm_field_name)
    django_value = getattr(django_obj, field_name)

    if field_name == 'password':
        modm_value = 'bcrypt${}'.format(modm_value)

    if type(django_value) != type(modm_value):
        # jsonb fields
        if modm_value is None and django_value == {}:
            return

        # bool defaulting to False but None in modm is falsey
        if modm_value is None and django_value is False:
            return

        # sometimes None is an empty string because MongoDB
        if modm_value is None and django_value == '':
            return

        if modm_value is None:
            logger.error('modm value was None but django value was {} for {} on {}.{}'.format(django_value, field_name, django_obj._meta.model.__module__, django_obj._meta.model.__class__))
            return

        # sometimes strings used to be integers, in some documents... LicenseRecord.year
        try:
            assert django_value.__class__(modm_value) == django_value, 'WITH TYPECAST: {}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, modm_value)
        except AssertionError as ex:
            logger.error(ex)
        return

    if isinstance(modm_value, datetime):
        try:
            assert django_value == pytz.utc.localize(modm_value), '{}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, pytz.utc.localize(modm_value))
        except AssertionError as ex:
            logger.error(ex)
        return
    try:
        assert django_value == modm_value, '{}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, modm_value)
    except AssertionError as ex:
        logger.error(ex)

def get_pk(modm_object, django_model, modm_to_django):
    try:
        return modm_to_django[format_lookup_key(modm_object._id, model=django_model)]
    except KeyError as ex:
        logger.error('modm key {} not found in lookup table {}'.format(
            format_lookup_key(modm_object._id, ContentType.objects.get_for_model(django_model).pk), ex))

@app.task(bind=True, max_retries=None)  # retry forever because of deadlocks
def validate_page_of_model_data(self, django_model, basic_fields, fk_relations, m2m_relations, offset, limit):
    try:
        set_backend()
        init_app(routes=False, attach_request_handlers=False, fixtures=False)
        register_nonexistent_models_with_modm()

        modm_model = get_modm_model(django_model)
        modm_to_django = build_toku_django_lookup_table_cache()

        if isinstance(django_model.modm_query, dict):
            modm_queryset = modm_model.find(**django_model.modm_query)
        else:
            modm_queryset = modm_model.find(django_model.modm_query)
        count = 0
        page_of_modm_objects = modm_queryset.sort('-_id')[offset:limit]
        django_ids = [get_pk(modm_obj, django_model, modm_to_django) for modm_obj in page_of_modm_objects]
        django_objects = django_model.objects.filter(id__in=django_ids)

        # TODO users aren't going to match
        if django_model is not OSFUser:
            try:
                assert len(django_ids) == len(django_objects) == len(page_of_modm_objects), 'Lost some keys along the way for {}.{}\n' \
                                                                                'Django_id count: {}\n' \
                                                                                'Django count: {}\n' \
                                                                                'Modm count: {}'.format(
                    django_model._meta.model.__module__,
                    django_model._meta.model.__name__,
                    len(django_ids),
                    len(django_objects),
                    len(page_of_modm_objects),
                )
            except AssertionError as ex:
                logger.error(ex)

        for modm_obj in page_of_modm_objects:
            try:
                django_obj = django_objects.get(pk=get_pk(modm_obj, django_model, modm_to_django))
            except ObjectDoesNotExist:
                logger.error('Django object {} with id {} does not exist'.format(django_model, modm_obj._id))
                continue
            for m2m_field in m2m_relations:
                validate_m2m_field(m2m_field.name, django_obj, modm_obj)
            for fk_field in fk_relations:
                validate_fk_relation(fk_field.name, django_obj, modm_obj)
            for basic_field in basic_fields:
                validate_basic_field(basic_field.name, django_obj, modm_obj)

            count += 1
        logger.info('Through {} {}.{}s...'.format(count, django_model._meta.model.__module__, django_model._meta.model.__name__))
    except Exception as ex:
        logger.error(
            'Retrying: Failed to validate page model: {} offset:{} limit:{} of foreign keys with exception {}'.format(
                django_model, offset, limit, ex))
        logger.error('{} {}'.format(ex.__dict__, ex.__class__))
        self.retry(countdown=20)  # retry in 20s

def validate_model_data(django_model, page_size=20000):
    logger.info('Starting {} on {}...'.format(
        sys._getframe().f_code.co_name, django_model._meta.model.__name__))
    count = 0
    modm_model = get_modm_model(django_model)
    if isinstance(django_model.modm_query, dict):
        modm_queryset = modm_model.find(**django_model.modm_query)
    else:
        modm_queryset = modm_model.find(django_model.modm_query)

    total = modm_queryset.count()

    m2m_relations = [field for field in
                     django_model._meta.get_fields() if
                     field.is_relation and field.many_to_many and not hasattr(field, 'field')]

    fk_relations = [field for field in django_model._meta.get_fields() if
                    field.is_relation and not field.auto_created and (field.many_to_one or field.one_to_one)]

    basic_fields = [field for field in django_model._meta.get_fields() if not field.is_relation]

    while count < total:
        offset = count
        limit = (count + page_size) if (count + page_size) < total else total
        validate_page_of_model_data.delay(django_model, basic_fields, fk_relations, m2m_relations, offset, limit)
        count += page_size
<commit_msg>More verification logic<commit_after>from __future__ import unicode_literals

import logging
import sys
from datetime import datetime

import pytz
from django.contrib.auth.models import Group, Permission
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist
from django.core.management import BaseCommand

from api.base.celery import app
from osf.management.commands.migratedata import register_nonexistent_models_with_modm, get_modm_model
from osf.management.commands.migraterelations import build_toku_django_lookup_table_cache, format_lookup_key
from osf.models import BlackListGuid, NotificationSubscription
from osf.models import EmbargoTerminationApproval
from osf.models import OSFUser
from osf.models import Tag
from osf.utils.order_apps import get_ordered_models
from website.app import init_app
from website.files.models import StoredFileNode
from .migratedata import set_backend
logger = logging.getLogger('migrations')

class NotGonnaDoItException(Exception):
    pass

# NodeLicenseRecord appears pretty broke
# StoredFileNode has no attribute deleted_by, probably deleted_on too

class Command(BaseCommand):
    help = 'Validates migrations from tokumx to postgres'

    def handle(self, *args, **options):
        django_models = get_ordered_models()

        for django_model in django_models:

            if not hasattr(django_model, 'modm_model_path'):
                logger.info('################################################\n'
                      '{} doesn\'t have a modm_model_path\n'
                      '################################################'.format(
                    django_model._meta.model.__name__))
                continue
            elif django_model.__subclasses__() != []:
                logger.info('Skipping {}.{}'.format(django_model.__module__, django_model))
                continue
            elif django_model is Tag or django_model is BlackListGuid:
                # TODO we'll do this by hand because tags are special
                # TODO we'll do blacklistguids by hand because they've got a bunch of new ones that don't exist in modm
                # TODO specifically, from register_none_existent_models
                continue
            do_model.delay(django_model)

@app.task()
def do_model(django_model):
    set_backend()
    init_app(routes=False, attach_request_handlers=False, fixtures=False)
    register_nonexistent_models_with_modm()

    page_size = django_model.migration_page_size

    validate_model_data(django_model, page_size=page_size)

def validate_m2m_field(field_name, django_obj, modm_obj):
    field = getattr(django_obj, field_name)

    # handle both forward and reverse rels
    if hasattr(django_obj, field_name):
        manager = field
        # skip groups and permissions
        if manager.model is Group or manager.model is Permission:
            return
        primary_identifier_name = manager.model.primary_identifier_name
    else:
        manager_name = '{}_set'.format(field.through._meta.object_name).lower()
        manager = getattr(django_obj, manager_name)
        try:
            primary_identifier_name = field.through._meta.model.primary_identifier_name
        except AttributeError:
            primary_identifier_name = '_id'
    django_guids = manager.all().values_list(primary_identifier_name, flat=True)
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    modm_guids = [obj._id for obj in getattr(modm_obj, modm_field_name)]
    for django_guid in django_guids:
        try:
            assert django_guid in modm_guids, '{} for model {}.{} was not in modm guids for field {}'.format(django_guid, django_obj._meta.model.__module__, django_obj._meta.model.__name__, modm_field_name)
        except AssertionError as ex:
            logger.error(ex)

def validate_fk_relation(field_name, django_obj, modm_obj):
    if field_name in ['content_type', 'content_type_pk', 'content_type_id']:
        # modm doesn't have gfk
        return

    if isinstance(modm_obj, StoredFileNode) and field_name == 'deleted_by':
        return

    if django_obj._meta.model is EmbargoTerminationApproval and field_name == 'initiated_by':
        # EmbargoTerminationApproval didn't have an initiated_by in modm even though it was supposed to.
        return

    django_field_value = getattr(django_obj, field_name)
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    if django_obj._meta.model is NotificationSubscription and field_name in ['user', 'node']:
        modm_field_name = 'owner'

    modm_field_value = getattr(modm_obj, modm_field_name)
    if modm_field_value and django_field_value:
        try:
            assert modm_field_value._id == django_field_value._id, 'Modm field {} of obj {}:{} with value of {} doesn\'t equal django field with value {}'.format(field_name, type(modm_obj), modm_obj._id, modm_field_value._id, django_field_value._id)
        except AssertionError as ex:
            logger.error(ex)
    elif modm_field_value is not None and django_field_value is None:
        logger.info('{} of {!r} was None in django but {} in modm'.format(field_name, modm_obj, django_obj, modm_field_value))

def validate_basic_field(field_name, django_obj, modm_obj):
    if field_name in ['id', 'pk', 'object_id', 'guid_string', 'last_login', 'is_superuser', 'content_type_pk', 'content_type_id']:
        # modm doesn't have these
        return
    # if there's a field alias, let's use that.
    if getattr(django_obj, 'FIELD_ALIASES', None) is None:
        modm_field_name = field_name
    else:
        modm_field_name = {v: k for k, v in getattr(django_obj, 'FIELD_ALIASES', {}).iteritems()}.get(field_name, field_name)

    if modm_field_name is False:
        return

    modm_value = getattr(modm_obj, modm_field_name)
    django_value = getattr(django_obj, field_name)

    if field_name == 'password':
        modm_value = 'bcrypt${}'.format(modm_value)

    if type(django_value) != type(modm_value):
        # jsonb fields
        if modm_value is None and django_value == {}:
            return

        # bool defaulting to False but None in modm is falsey
        if modm_value is None and django_value is False:
            return

        # sometimes None is an empty string because MongoDB
        if modm_value is None and django_value == '':
            return

        if modm_value is None:
            logger.error('modm value was None but django value was {} for {} on {}.{}'.format(django_value, field_name, django_obj._meta.model.__module__, django_obj._meta.model.__class__))
            return

        # sometimes strings used to be integers, in some documents... LicenseRecord.year
        try:
            assert django_value.__class__(modm_value) == django_value, 'WITH TYPECAST: {}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, modm_value)
        except AssertionError as ex:
            logger.error(ex)
        return

    if isinstance(modm_value, datetime):
        try:
            assert django_value == pytz.utc.localize(modm_value), '{}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, pytz.utc.localize(modm_value))
        except AssertionError as ex:
            logger.error(ex)
        return
    try:
        assert django_value == modm_value, '{}.{} {} {}:{} {}:{}'.format(django_obj.__module__, django_obj.__class__, django_obj, field_name, django_value, modm_field_name, modm_value)
    except AssertionError as ex:
        logger.error(ex)

def get_pk(modm_object, django_model, modm_to_django):
    try:
        return modm_to_django[format_lookup_key(modm_object._id, model=django_model)]
    except KeyError as ex:
        logger.error('modm key {} not found in lookup table {}'.format(
            format_lookup_key(modm_object._id, ContentType.objects.get_for_model(django_model).pk), ex))

@app.task(bind=True, max_retries=None)  # retry forever because of deadlocks
def validate_page_of_model_data(self, django_model, basic_fields, fk_relations, m2m_relations, offset, limit):
    try:
        set_backend()
        init_app(routes=False, attach_request_handlers=False, fixtures=False)
        register_nonexistent_models_with_modm()

        modm_model = get_modm_model(django_model)
        modm_to_django = build_toku_django_lookup_table_cache()

        if isinstance(django_model.modm_query, dict):
            modm_queryset = modm_model.find(**django_model.modm_query)
        else:
            modm_queryset = modm_model.find(django_model.modm_query)
        count = 0
        page_of_modm_objects = modm_queryset.sort('-_id')[offset:limit]
        django_ids = [get_pk(modm_obj, django_model, modm_to_django) for modm_obj in page_of_modm_objects]
        django_objects = django_model.objects.filter(id__in=django_ids)

        # TODO users aren't going to match
        if django_model is not OSFUser:
            try:
                assert len(django_ids) == len(django_objects) == len(page_of_modm_objects), 'Lost some keys along the way for {}.{}\n' \
                                                                                'Django_id count: {}\n' \
                                                                                'Django count: {}\n' \
                                                                                'Modm count: {}'.format(
                    django_model._meta.model.__module__,
                    django_model._meta.model.__name__,
                    len(django_ids),
                    len(django_objects),
                    len(page_of_modm_objects),
                )
            except AssertionError as ex:
                logger.error(ex)

        for modm_obj in page_of_modm_objects:
            try:
                django_obj = django_objects.get(pk=get_pk(modm_obj, django_model, modm_to_django))
            except ObjectDoesNotExist:
                logger.error('Django object {} with id {} does not exist'.format(django_model, modm_obj._id))
                continue
            for m2m_field in m2m_relations:
                validate_m2m_field(m2m_field.name, django_obj, modm_obj)
            for fk_field in fk_relations:
                validate_fk_relation(fk_field.name, django_obj, modm_obj)
            for basic_field in basic_fields:
                validate_basic_field(basic_field.name, django_obj, modm_obj)

            count += 1
        logger.info('Through {} {}.{}s...'.format(count, django_model._meta.model.__module__, django_model._meta.model.__name__))
    except Exception as ex:
        logger.error(
            'Retrying: Failed to validate page model: {} offset:{} limit:{} of foreign keys with exception {}'.format(
                django_model, offset, limit, ex))
        logger.error('{} {}'.format(ex.__dict__, ex.__class__))
        self.retry(countdown=20)  # retry in 20s

def validate_model_data(django_model, page_size=20000):
    logger.info('Starting {} on {}...'.format(
        sys._getframe().f_code.co_name, django_model._meta.model.__name__))
    count = 0
    modm_model = get_modm_model(django_model)
    if isinstance(django_model.modm_query, dict):
        modm_queryset = modm_model.find(**django_model.modm_query)
    else:
        modm_queryset = modm_model.find(django_model.modm_query)

    total = modm_queryset.count()

    m2m_relations = [field for field in
                     django_model._meta.get_fields() if
                     field.is_relation and field.many_to_many and not hasattr(field, 'field')]

    fk_relations = [field for field in django_model._meta.get_fields() if
                    field.is_relation and not field.auto_created and (field.many_to_one or field.one_to_one)]

    basic_fields = [field for field in django_model._meta.get_fields() if not field.is_relation]

    while count < total:
        offset = count
        limit = (count + page_size) if (count + page_size) < total else total
        validate_page_of_model_data.delay(django_model, basic_fields, fk_relations, m2m_relations, offset, limit)
        count += page_size
<|endoftext|>


<commit_before># Copyright 2015 Rackspace Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import fixtures
import functools
import logging as pylogging
import mock
import platform
import time

from oslo_log import formatters
from oslo_log import log as logging
from oslotest import base
import testtools

from oslo_privsep import capabilities
from oslo_privsep import daemon
from oslo_privsep.tests import testctx

LOG = logging.getLogger(__name__)

def undecorated():
    pass

class TestException(Exception):
    pass

@testctx.context.entrypoint
def logme(level, msg, exc_info=False):
    # We want to make sure we log everything from the priv side for
    # the purposes of this test, so force loglevel.
    LOG.logger.setLevel(logging.DEBUG)
    if exc_info:
        try:
            raise TestException('with arg')
        except TestException:
            LOG.log(level, msg, exc_info=True)
    else:
        LOG.log(level, msg)

class LogRecorder(pylogging.Formatter):
    def __init__(self, logs, *args, **kwargs):
        super(LogRecorder, self).__init__(*args, **kwargs)
        self.logs = logs

    def format(self, record):
        self.logs.append(copy.deepcopy(record))
        return super(LogRecorder, self).format(record)

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class LogTest(testctx.TestContextTestCase):
    def setUp(self):
        super(LogTest, self).setUp()

    def test_priv_loglevel(self):
        logger = self.useFixture(fixtures.FakeLogger(
            level=logging.INFO))

        # These write to the log on the priv side
        logme(logging.DEBUG, u'test@DEBUG')
        logme(logging.WARN, u'test@WARN')

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        # logger.output is the resulting log on the unpriv side.
        # This should have been filtered based on (unpriv) loglevel.
        self.assertNotIn(u'test@DEBUG', logger.output)
        self.assertIn(u'test@WARN', logger.output)

    def test_record_data(self):
        logs = []

        self.useFixture(fixtures.FakeLogger(
            level=logging.INFO, format='dummy',
            # fixtures.FakeLogger accepts only a formatter
            # class/function, not an instance :(
            formatter=functools.partial(LogRecorder, logs)))

        logme(logging.WARN, u'test with exc', exc_info=True)

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        self.assertEqual(1, len(logs))

        record = logs[0]
        self.assertIn(u'test with exc', record.getMessage())
        self.assertEqual(None, record.exc_info)
        self.assertIn(u'TestException: with arg', record.exc_text)
        self.assertEqual('PrivContext(cfg_section=privsep)',
                         record.processName)
        self.assertIn(u'test_daemon.py', record.exc_text)
        self.assertEqual(logging.WARN, record.levelno)
        self.assertEqual('logme', record.funcName)

    def test_format_record(self):
        logs = []

        self.useFixture(fixtures.FakeLogger(
            level=logging.INFO, format='dummy',
            # fixtures.FakeLogger accepts only a formatter
            # class/function, not an instance :(
            formatter=functools.partial(LogRecorder, logs)))

        logme(logging.WARN, u'test with exc', exc_info=True)

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        self.assertEqual(1, len(logs))

        record = logs[0]
        # Verify the log record can be formatted by ContextFormatter
        fake_config = mock.Mock(
            logging_default_format_string="NOCTXT: %(message)s")
        formatter = formatters.ContextFormatter(config=fake_config)
        formatter.format(record)

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class DaemonTest(base.BaseTestCase):

    @mock.patch('os.setuid')
    @mock.patch('os.setgid')
    @mock.patch('os.setgroups')
    @mock.patch('oslo_privsep.capabilities.set_keepcaps')
    @mock.patch('oslo_privsep.capabilities.drop_all_caps_except')
    def test_drop_privs(self, mock_dropcaps, mock_keepcaps,
                        mock_setgroups, mock_setgid, mock_setuid):
        channel = mock.NonCallableMock()
        context = mock.NonCallableMock()
        context.conf.user = 42
        context.conf.group = 84
        context.conf.capabilities = [
            capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN]

        d = daemon.Daemon(channel, context)
        d._drop_privs()

        mock_setuid.assert_called_once_with(42)
        mock_setgid.assert_called_once_with(84)
        mock_setgroups.assert_called_once_with([])

        self.assertItemsEqual(
            [mock.call(True), mock.call(False)],
            mock_keepcaps.mock_calls)

        mock_dropcaps.assert_called_once_with(
            set((capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN)),
            set((capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN)),
            [])

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class WithContextTest(testctx.TestContextTestCase):

    def test_unexported(self):
        self.assertRaisesRegexp(
            NameError, 'undecorated not exported',
            testctx.context._wrap, undecorated)
<commit_msg>Using assertIsNone(xxx) instead of assertEqual(None, xxx)<commit_after># Copyright 2015 Rackspace Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import fixtures
import functools
import logging as pylogging
import mock
import platform
import time

from oslo_log import formatters
from oslo_log import log as logging
from oslotest import base
import testtools

from oslo_privsep import capabilities
from oslo_privsep import daemon
from oslo_privsep.tests import testctx

LOG = logging.getLogger(__name__)

def undecorated():
    pass

class TestException(Exception):
    pass

@testctx.context.entrypoint
def logme(level, msg, exc_info=False):
    # We want to make sure we log everything from the priv side for
    # the purposes of this test, so force loglevel.
    LOG.logger.setLevel(logging.DEBUG)
    if exc_info:
        try:
            raise TestException('with arg')
        except TestException:
            LOG.log(level, msg, exc_info=True)
    else:
        LOG.log(level, msg)

class LogRecorder(pylogging.Formatter):
    def __init__(self, logs, *args, **kwargs):
        super(LogRecorder, self).__init__(*args, **kwargs)
        self.logs = logs

    def format(self, record):
        self.logs.append(copy.deepcopy(record))
        return super(LogRecorder, self).format(record)

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class LogTest(testctx.TestContextTestCase):
    def setUp(self):
        super(LogTest, self).setUp()

    def test_priv_loglevel(self):
        logger = self.useFixture(fixtures.FakeLogger(
            level=logging.INFO))

        # These write to the log on the priv side
        logme(logging.DEBUG, u'test@DEBUG')
        logme(logging.WARN, u'test@WARN')

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        # logger.output is the resulting log on the unpriv side.
        # This should have been filtered based on (unpriv) loglevel.
        self.assertNotIn(u'test@DEBUG', logger.output)
        self.assertIn(u'test@WARN', logger.output)

    def test_record_data(self):
        logs = []

        self.useFixture(fixtures.FakeLogger(
            level=logging.INFO, format='dummy',
            # fixtures.FakeLogger accepts only a formatter
            # class/function, not an instance :(
            formatter=functools.partial(LogRecorder, logs)))

        logme(logging.WARN, u'test with exc', exc_info=True)

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        self.assertEqual(1, len(logs))

        record = logs[0]
        self.assertIn(u'test with exc', record.getMessage())
        self.assertIsNone(record.exc_info)
        self.assertIn(u'TestException: with arg', record.exc_text)
        self.assertEqual('PrivContext(cfg_section=privsep)',
                         record.processName)
        self.assertIn(u'test_daemon.py', record.exc_text)
        self.assertEqual(logging.WARN, record.levelno)
        self.assertEqual('logme', record.funcName)

    def test_format_record(self):
        logs = []

        self.useFixture(fixtures.FakeLogger(
            level=logging.INFO, format='dummy',
            # fixtures.FakeLogger accepts only a formatter
            # class/function, not an instance :(
            formatter=functools.partial(LogRecorder, logs)))

        logme(logging.WARN, u'test with exc', exc_info=True)

        time.sleep(0.1)  # Hack to give logging thread a chance to run

        self.assertEqual(1, len(logs))

        record = logs[0]
        # Verify the log record can be formatted by ContextFormatter
        fake_config = mock.Mock(
            logging_default_format_string="NOCTXT: %(message)s")
        formatter = formatters.ContextFormatter(config=fake_config)
        formatter.format(record)

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class DaemonTest(base.BaseTestCase):

    @mock.patch('os.setuid')
    @mock.patch('os.setgid')
    @mock.patch('os.setgroups')
    @mock.patch('oslo_privsep.capabilities.set_keepcaps')
    @mock.patch('oslo_privsep.capabilities.drop_all_caps_except')
    def test_drop_privs(self, mock_dropcaps, mock_keepcaps,
                        mock_setgroups, mock_setgid, mock_setuid):
        channel = mock.NonCallableMock()
        context = mock.NonCallableMock()
        context.conf.user = 42
        context.conf.group = 84
        context.conf.capabilities = [
            capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN]

        d = daemon.Daemon(channel, context)
        d._drop_privs()

        mock_setuid.assert_called_once_with(42)
        mock_setgid.assert_called_once_with(84)
        mock_setgroups.assert_called_once_with([])

        self.assertItemsEqual(
            [mock.call(True), mock.call(False)],
            mock_keepcaps.mock_calls)

        mock_dropcaps.assert_called_once_with(
            set((capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN)),
            set((capabilities.CAP_SYS_ADMIN, capabilities.CAP_NET_ADMIN)),
            [])

@testtools.skipIf(platform.system() != 'Linux',
                  'works only on Linux platform.')
class WithContextTest(testctx.TestContextTestCase):

    def test_unexported(self):
        self.assertRaisesRegexp(
            NameError, 'undecorated not exported',
            testctx.context._wrap, undecorated)
<|endoftext|>


<commit_before>__author__ = 'kemi'

# Built in Imports
import requests
import platform
import tempfile

# Cloudify Imports
from package_installer_plugin.constants import *
from utils import run
from cloudify import ctx
from cloudify import exceptions
from cloudify.decorators import operation

@operation
def install_packages(config, **_):
    """ Installs a list of packages """

    ctx.logger.info('Attempting to install packages')
    distro = platform.linux_distribution(full_distribution_name=False)
    distro_lower = [x.lower() for x in distro]
    ctx.logger.info('Working environment: {0} {2} v{1}'.format(distro[0], distro[1], distro[2]))

    ctx.logger.info(config)

    if 'custom_repo' in config:
        _add_custom_repo(config['custom_repo'], distro_lower)

    package_list = config['package_list']
    for package_to_install in package_list:

        # Install from repository (yum / apt)
        if 'http' not in package_to_install:
            ctx.logger.info('Installing from repository: {0}'.format(package_to_install))

            install_command = _get_install_command(
                distro=distro_lower,
                install_from_repo=True,
                package=package_to_install)

            # TODO: apt-get update should not get called every install
            run(apt_get_update)

        # Install from package
        else:
            ctx.logger.info('Installing from URL: {0}'.format(package_to_install))

            _, package_file = tempfile.mkstemp()
            _download_package(package_file, package_to_install)

            install_command = _get_install_command(
                distro=distro_lower,
                install_from_repo=False,
                package=package_to_install)

        ctx.logger.info('Running command: {0}'.format(install_command))
        run(install_command)

def _add_custom_repo(repo, distro):
    if 'ubuntu' in distro:
        key_server = repo['apt', 'key_server']
        add_key_server_command = 'sudo apt-key adv --keyserver ' + key_server
        run(add_key_server_command)
        add_repo_cmd = repo['apt', 'list_file']
        run(add_repo_cmd)
    elif 'centos' in distro:
        repo_name = repo['yum', 'name']
        repo_entry = repo['yum', 'entry']
        add_repo_cmd = 'echo ' + repo_entry + ' | sudo tee /etc/yum.repos.d/{0}.repo'.format(repo_name)
    else:
        raise exceptions.NonRecoverableError(
            'Only CentOS and Ubuntu supported.')
    run(add_repo_cmd)

def _get_install_command(distro, install_from_repo, package):
    if 'ubuntu' in distro:
        if install_from_repo:
            install_command = apt_get + install + '{0}'.format(package)
        else:
            install_command = dpkg + '{0}'.format(package)
        ctx.logger.info('Installing on Ubuntu: ' + install_command)
    elif 'centos' in distro:
        if install_from_repo:
            install_command = yum + install + package
        else:
            install_command = yum + install + '{0}'.format(package)
        ctx.logger.info('Installing on CentOS: ' + install_command)
    else:
        raise exceptions.NonRecoverableError(
            'Only CentOS and Ubuntu supported.')

    return install_command

def _download_package(package_file, url):
    """ Downloads package from url to tempfile """

    ctx.logger.debug('Downloading: {0}'.format(url))
    package = requests.get(url, stream=True)

    with open(package_file, 'wb') as f:
        for chunk in package.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                f.flush()

@operation
def remove_package(package_list, **kwargs):
    """ removes a package from a given node """

    if 'ubuntu' in platform:
        # TODO: apt-get update should not get called every install
        run(apt_get_update)
        remove_command = apt_get + remove + '{0}'.format(package_list)
    elif 'centos' in platform:
        remove_command = yum + remove + '{0}'.format(package_list)
    else:
        raise exceptions.NonRecoverableError(
            'Only Centos and Ubuntu supported.')
    run(remove_command)<commit_msg>correct check<commit_after>__author__ = 'kemi'

# Built in Imports
import requests
import platform
import tempfile

# Cloudify Imports
from package_installer_plugin.constants import *
from utils import run
from cloudify import ctx
from cloudify import exceptions
from cloudify.decorators import operation

@operation
def install_packages(config, **_):
    """ Installs a list of packages """

    ctx.logger.info('Attempting to install packages')
    distro = platform.linux_distribution(full_distribution_name=False)
    distro_lower = [x.lower() for x in distro]
    ctx.logger.info('Working environment: {0} {2} v{1}'.format(distro[0], distro[1], distro[2]))

    ctx.logger.info(config)

    if 'custom_repo' in config:
        _add_custom_repo(config['custom_repo'], distro_lower)

    package_list = config['package_list']
    for package_to_install in package_list:

        # Install from repository (yum / apt)
        if 'http' not in package_to_install:
            ctx.logger.info('Installing from repository: {0}'.format(package_to_install))

            install_command = _get_install_command(
                distro=distro_lower,
                install_from_repo=True,
                package=package_to_install)

            # TODO: apt-get update should not get called every install
            run(apt_get_update)

        # Install from package
        else:
            ctx.logger.info('Installing from URL: {0}'.format(package_to_install))

            _, package_file = tempfile.mkstemp()
            _download_package(package_file, package_to_install)

            install_command = _get_install_command(
                distro=distro_lower,
                install_from_repo=False,
                package=package_to_install)

        ctx.logger.info('Running command: {0}'.format(install_command))
        run(install_command)

def _add_custom_repo(repo, distro):
    if 'ubuntu' in distro:
        key_server = repo['apt', 'key_server']
        add_key_server_command = 'sudo apt-key adv --keyserver ' + key_server
        ctx.logger.info('Adding key server: {0}'.format(add_key_server_command))
        run(add_key_server_command)
        add_repo_cmd = repo['apt', 'list_file']
    elif 'centos' in distro:
        repo_name = repo['yum', 'name']
        repo_entry = repo['yum', 'entry']
        add_repo_cmd = 'echo ' + repo_entry + ' | sudo tee /etc/yum.repos.d/{0}.repo'.format(repo_name)
    else:
        raise exceptions.NonRecoverableError(
            'Only CentOS and Ubuntu supported.')
    ctx.logger.info('Running command: {0}'.format(add_repo_cmd))
    run(add_repo_cmd)

def _get_install_command(distro, install_from_repo, package):
    if 'ubuntu' in distro:
        if install_from_repo:
            install_command = apt_get + install + '{0}'.format(package)
        else:
            install_command = dpkg + '{0}'.format(package)
        ctx.logger.info('Installing on Ubuntu: ' + install_command)
    elif 'centos' in distro:
        if install_from_repo:
            install_command = yum + install + package
        else:
            install_command = yum + install + '{0}'.format(package)
        ctx.logger.info('Installing on CentOS: ' + install_command)
    else:
        raise exceptions.NonRecoverableError(
            'Only CentOS and Ubuntu supported.')

    return install_command

def _download_package(package_file, url):
    """ Downloads package from url to tempfile """

    ctx.logger.debug('Downloading: {0}'.format(url))
    package = requests.get(url, stream=True)

    with open(package_file, 'wb') as f:
        for chunk in package.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                f.flush()

@operation
def remove_package(package_list, **kwargs):
    """ removes a package from a given node """

    if 'ubuntu' in platform:
        # TODO: apt-get update should not get called every install
        run(apt_get_update)
        remove_command = apt_get + remove + '{0}'.format(package_list)
    elif 'centos' in platform:
        remove_command = yum + remove + '{0}'.format(package_list)
    else:
        raise exceptions.NonRecoverableError(
            'Only Centos and Ubuntu supported.')
    run(remove_command)<|endoftext|>


<commit_before>#! /usr/bin/env python
#
# Tests sending a push notification via adb to The Blue Alliance Android app.
# This tests the app's notification response without having to make a debug TBA
# server on App Engine, set up GCM credentials, build a custom GCM RPC library,
# temporarily modify the app code for it, and other steps.
#
# This requires a Debug build of TBA Android which turns off the sender-
# permission requirement for its GCM IntentReceiver, thus accepting Intents
# from the Activity Manager tool "am".
# http://stackoverflow.com/questions/25931902/is-it-possible-to-simulate-a-gcm-receive-from-the-adb-shell-am-command-line-i/29425669#29425669
#
# NOTE: The shell quoting only works if "am broadcast ... 'JSON_TEXT'" is *one
# argument* to the adb shell command.
# http://stackoverflow.com/questions/27940118/sending-json-as-extra-data-in-an-android-broadcast-via-adb-gets-incorrectly-form/29428061#29428061
#
#
# Using this script from the command line (as opposed to importing it into
# another Python program) requires: pip install scriptine
#
# For usage info: "test_notification.py --help",
#     "test_notification.py upcoming_match -h", ...

import subprocess, json

# Sends a test push-notification over adb to TBA Android app.
#
# json can be a json-encoded string or a Python dict to json-encode.
#
# This should echo something like:
#    Broadcasting: Intent { act=com.google.android.c2dm.intent.RECEIVE cat=[com.thebluealliance.androidclient] (has extras) }
#    Broadcast completed: result=-1
def notify(message_type, json_data):
    if type(json_data) != str:
        json_data = json.dumps(json_data)

    # Quoting for "Chairman's".
    json_text = json_data.replace("'", """'"'"'""")

    template = """am broadcast -a com.google.android.c2dm.intent.RECEIVE \
        -c com.thebluealliance.androidclient \
        --es message_type %s \
        --es message_data '%s'"""
    command = template % (message_type, json_text)

    subprocess.call(["adb", "shell", command])

# ====== scriptine commands ======

upcoming_match_sample = {
    "match_key": "2014calb_qm17",
    "event_name": "Los Angeles Regional sponsored by The Roddenberry Foundation",
    "team_keys": ["frc971","frc148","frc254","frc1983","frc1114","frc1318"],
    "scheduled_time": 12345,
    "predicted_time": 123456
}

def upcoming_match_command(data=upcoming_match_sample):
    notify('upcoming_match', data)

match_score_sample = {
    "event_name": "New England FRC Region Championship",
    "match": {
      "comp_level": "f",
      "match_number": 1,
      "videos": [],
      "time_string": "3:18 PM",
      "set_number": 1,
      "key": "2014necmp_f1m1",
      "time": 1397330280,
      "score_breakdown": None,
      "alliances": {
        "blue": {"score": 154, "teams": ["frc177","frc230","frc4055"]},
        "red": {"score": 78, "teams": ["frc195","frc558","frc5122"]}
      },
      "event_key": "2014necmp"
    }
}

def match_score_command(data=match_score_sample):
    notify('match_score', data)

level_starting_sample = {
    "event_name": "Hawaii Regional",
    "comp_level": "f",
    "event_key": "2014hiho",
    "scheduled_time": 1397330280
}

def level_starting_command(data=level_starting_sample):
    notify('starting_comp_level', data)

alliance_selection_sample = {
    "event": {
      "key": "2014necmp",
      "end_date": "2014-04-12",
      "name": "New England FRC Region Championship",
      "short_name": "New England",
      "event_district_string": "New England",
      "event_district": 3,
      "location": "Boston, MA, USA",
      "event_code": "necmp",
      "year": 2014,
      "alliances": [
        {"picks": ["frc195","frc558","frc5122"]},
        {"picks": ["frc1153","frc125","frc4048"]},
        {"picks": ["frc230","frc177","frc4055"]},
        {"picks": ["frc716","frc78","frc811"]},
        {"picks": ["frc1519","frc3467","frc58"]},
        {"picks": ["frc131","frc175","frc1073"]},
        {"picks": ["frc228","frc3525","frc2168"]},
        {"picks": ["frc172","frc1058","frc2067"]}
      ],
      "event_type_string": "District Championship",
      "start_date": "2014-04-10",
      "event_type": 2
    }
}

def alliance_selection_command(data=alliance_selection_sample):
    notify('alliance_selection', data)

awards_posted_sample = {
    "event_name": "New England FRC Region Championship",
    "event_key": "2014necmp",
    "awards": [
      {
        "award_type": 0,
        "name": "Regional Chairman's Award",
        "recipient_list": [
          {"team_number": 2067, "awardee": None},
          {"team_number": 78, "awardee": None},
          {"team_number": 811, "awardee": None},
          {"team_number": 2648, "awardee": None}
        ],
        "year": 2014
      }
    ]
}

def awards_posted_command(data=awards_posted_sample):
    notify('awards_posted', data)

schedule_updated_sample = {
    "event_name": "Australia Regional",
    "first_match_time": 1397330280,
    "event_key": "2015ausy"
}

def schedule_updated_command(data=schedule_updated_sample):
    notify('schedule_updated', data)

district_points_updated_sample = {
    "district_name": "Pacific Northwest",
    "district_key": "2014pnw"
}

def district_points_updated_command(data=district_points_updated_sample):
    notify('district_points_updated', data)

ping_sample = {
    "title": "TBA Test Message",
    "desc": "This is a test message ensuring your device can receive push messages from The Blue Alliance",
    "url": "https://www.youtube.com/watch?v=RpSgUrsghv4"
}

def ping_command(data=ping_sample, url="", no_url=False):
    if url:
        data["url"] = url

    if no_url:
        del data["url"]

    notify('ping', data)

def broadcast_command(data=ping_sample):
    if url:
        data["url"] = url

    if no_url:
        del data["url"]

    notify('broadcast', data)

sync_status_sample = {}

def sync_status_command(data=sync_status_sample):
    notify('sync_status', data)

event_down_sample = {
    "event_key": "2015cthar",
    "event_name": "Hartford"
}

def event_down_command(data=event_down_sample):
    notify('event_down', data)

# ====== main ======

if __name__ == '__main__':
    import scriptine
    scriptine.run()
<commit_msg>Added missing arguments to the broadcast command<commit_after>#! /usr/bin/env python
#
# Tests sending a push notification via adb to The Blue Alliance Android app.
# This tests the app's notification response without having to make a debug TBA
# server on App Engine, set up GCM credentials, build a custom GCM RPC library,
# temporarily modify the app code for it, and other steps.
#
# This requires a Debug build of TBA Android which turns off the sender-
# permission requirement for its GCM IntentReceiver, thus accepting Intents
# from the Activity Manager tool "am".
# http://stackoverflow.com/questions/25931902/is-it-possible-to-simulate-a-gcm-receive-from-the-adb-shell-am-command-line-i/29425669#29425669
#
# NOTE: The shell quoting only works if "am broadcast ... 'JSON_TEXT'" is *one
# argument* to the adb shell command.
# http://stackoverflow.com/questions/27940118/sending-json-as-extra-data-in-an-android-broadcast-via-adb-gets-incorrectly-form/29428061#29428061
#
#
# Using this script from the command line (as opposed to importing it into
# another Python program) requires: pip install scriptine
#
# For usage info: "test_notification.py --help",
#     "test_notification.py upcoming_match -h", ...

import subprocess, json

# Sends a test push-notification over adb to TBA Android app.
#
# json can be a json-encoded string or a Python dict to json-encode.
#
# This should echo something like:
#    Broadcasting: Intent { act=com.google.android.c2dm.intent.RECEIVE cat=[com.thebluealliance.androidclient] (has extras) }
#    Broadcast completed: result=-1
def notify(message_type, json_data):
    if type(json_data) != str:
        json_data = json.dumps(json_data)

    # Quoting for "Chairman's".
    json_text = json_data.replace("'", """'"'"'""")

    template = """am broadcast -a com.google.android.c2dm.intent.RECEIVE \
        -c com.thebluealliance.androidclient \
        --es message_type %s \
        --es message_data '%s'"""
    command = template % (message_type, json_text)

    subprocess.call(["adb", "shell", command])

# ====== scriptine commands ======

upcoming_match_sample = {
    "match_key": "2014calb_qm17",
    "event_name": "Los Angeles Regional sponsored by The Roddenberry Foundation",
    "team_keys": ["frc971","frc148","frc254","frc1983","frc1114","frc1318"],
    "scheduled_time": 12345,
    "predicted_time": 123456
}

def upcoming_match_command(data=upcoming_match_sample):
    notify('upcoming_match', data)

match_score_sample = {
    "event_name": "New England FRC Region Championship",
    "match": {
      "comp_level": "f",
      "match_number": 1,
      "videos": [],
      "time_string": "3:18 PM",
      "set_number": 1,
      "key": "2014necmp_f1m1",
      "time": 1397330280,
      "score_breakdown": None,
      "alliances": {
        "blue": {"score": 154, "teams": ["frc177","frc230","frc4055"]},
        "red": {"score": 78, "teams": ["frc195","frc558","frc5122"]}
      },
      "event_key": "2014necmp"
    }
}

def match_score_command(data=match_score_sample):
    notify('match_score', data)

level_starting_sample = {
    "event_name": "Hawaii Regional",
    "comp_level": "f",
    "event_key": "2014hiho",
    "scheduled_time": 1397330280
}

def level_starting_command(data=level_starting_sample):
    notify('starting_comp_level', data)

alliance_selection_sample = {
    "event": {
      "key": "2014necmp",
      "end_date": "2014-04-12",
      "name": "New England FRC Region Championship",
      "short_name": "New England",
      "event_district_string": "New England",
      "event_district": 3,
      "location": "Boston, MA, USA",
      "event_code": "necmp",
      "year": 2014,
      "alliances": [
        {"picks": ["frc195","frc558","frc5122"]},
        {"picks": ["frc1153","frc125","frc4048"]},
        {"picks": ["frc230","frc177","frc4055"]},
        {"picks": ["frc716","frc78","frc811"]},
        {"picks": ["frc1519","frc3467","frc58"]},
        {"picks": ["frc131","frc175","frc1073"]},
        {"picks": ["frc228","frc3525","frc2168"]},
        {"picks": ["frc172","frc1058","frc2067"]}
      ],
      "event_type_string": "District Championship",
      "start_date": "2014-04-10",
      "event_type": 2
    }
}

def alliance_selection_command(data=alliance_selection_sample):
    notify('alliance_selection', data)

awards_posted_sample = {
    "event_name": "New England FRC Region Championship",
    "event_key": "2014necmp",
    "awards": [
      {
        "award_type": 0,
        "name": "Regional Chairman's Award",
        "recipient_list": [
          {"team_number": 2067, "awardee": None},
          {"team_number": 78, "awardee": None},
          {"team_number": 811, "awardee": None},
          {"team_number": 2648, "awardee": None}
        ],
        "year": 2014
      }
    ]
}

def awards_posted_command(data=awards_posted_sample):
    notify('awards_posted', data)

schedule_updated_sample = {
    "event_name": "Australia Regional",
    "first_match_time": 1397330280,
    "event_key": "2015ausy"
}

def schedule_updated_command(data=schedule_updated_sample):
    notify('schedule_updated', data)

district_points_updated_sample = {
    "district_name": "Pacific Northwest",
    "district_key": "2014pnw"
}

def district_points_updated_command(data=district_points_updated_sample):
    notify('district_points_updated', data)

ping_sample = {
    "title": "TBA Test Message",
    "desc": "This is a test message ensuring your device can receive push messages from The Blue Alliance",
    "url": "https://www.youtube.com/watch?v=RpSgUrsghv4"
}

def ping_command(data=ping_sample, url="", no_url=False):
    if url:
        data["url"] = url

    if no_url:
        del data["url"]

    notify('ping', data)

def broadcast_command(data=ping_sample, url="", no_url=False):
    if url:
        data["url"] = url

    if no_url:
        del data["url"]

    notify('broadcast', data)

sync_status_sample = {}

def sync_status_command(data=sync_status_sample):
    notify('sync_status', data)

event_down_sample = {
    "event_key": "2015cthar",
    "event_name": "Hartford"
}

def event_down_command(data=event_down_sample):
    notify('event_down', data)

# ====== main ======

if __name__ == '__main__':
    import scriptine
    scriptine.run()
<|endoftext|>