<commit_before>""" Testing IVIM

"""
import numpy as np
from numpy.testing import (assert_array_equal, assert_array_almost_equal)

from dipy.reconst.ivim import ivim_function, IvimModel
from dipy.core.gradients import gradient_table
from dipy.sims.voxel import multi_tensor
from dipy.core.sphere import disperse_charges, HemiSphere

def test_fit_minimize():
    """
    Test the implementation of the fitting with minimize
    """
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)

    S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

    mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
    # This gives an isotropic signal

    signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
                          f * 100, 100 * (1 - f)])
    data = np.array([signal[0], ])

    guess = np.array([[1.0, 0.10, 0.001, 0.0001], ])
    ivim_model = IvimModel(gtab)
    ivim_fit = ivim_model.fit(data, routine="minimize", x0=guess, tol=1e-7)

    est_signal = np.array([ivim_function(ivim_fit.model_params[0], bvals), ])

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params[0], [S0, f, D_star, D])

def test_fit_leastsq():
    """
    Test the implementation of the fitting with leastsq
    """
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)

    S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

    mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
    # This gives an isotropic signal

    signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
                          f * 100, 100 * (1 - f)])
    data = signal[0]
    ivim_model = IvimModel(gtab)
    ivim_fit = ivim_model.fit(data, routine="leastsq")

    est_signal = ivim_function(ivim_fit.model_params, bvals)

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params, [S0, f, D_star, D])

def test_multivoxel():
    """Test fitting with multivoxel data"""
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)
    params = [[1.0, 0.06, 0.0072, 0.00097], [9.0, 0.05, 0.0074, 0.00087]]

    data = generate_multivoxel_data(gtab, params)
    ivim_model = IvimModel(gtab)

    guess_params = np.array([[1.0, 0.01, 0.001, 0.0009],
                             [0.9, 0.04, 0.002, 0.0004]])
    ivim_fit = ivim_model.fit(data, x0=guess_params)
    est_signal = generate_multivoxel_data(gtab, ivim_fit.model_params)

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params, params)

def get_bvecs(N):
    """Generate bvectors for N bvalues"""
    theta = np.pi * np.random.rand(N)
    phi = 2 * np.pi * np.random.rand(N)
    hsph_initial = HemiSphere(theta=theta, phi=phi)
    hsph_updated, potential = disperse_charges(hsph_initial, 5000)
    vertices = hsph_updated.vertices
    return vertices

def generate_multivoxel_data(gtab, params):
    """Generate multivoxel data for testing"""
    data = []
    for parameters in params:
        S0, f, D_star, D = parameters
        mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
        signal = multi_tensor(gtab, mevals, S0=S0, snr=None, fractions=[
            f * 100, 100 * (1 - f)])
        data.append(signal[0])
    return np.array(data)

def test_two_stage():
    """Test the two stage fitting routine"""
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)

    S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

    mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
    # This gives an isotropic signal

    signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
                          f * 100, 100 * (1 - f)])
    data = signal[0]
    ivim_model = IvimModel(gtab)

    guess_params = np.array([[1.0, .10, 0.001, 0.0001], ])
    ivim_fit = ivim_model.fit(data, fit_method="two_stage", routine='minimize')

    est_signal = ivim_function(ivim_fit.model_params, bvals)

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params, [S0, f, D_star, D])

    # Test with multiple voxels

    params = [[1.0, 0.06, 0.0072, 0.00097], [9.0, 0.05, 0.0074, 0.00087]]

    data_two = generate_multivoxel_data(gtab, params)
    ivim_model_two = IvimModel(gtab)

    guess_params = np.array([[1.0, 0.01, 0.001, 0.0009],
                             [0.9, 0.04, 0.002, 0.0004]])
    ivim_fit_two = ivim_model_two.fit(data_two, fit_method="two_stage",
                                      x0=guess_params)
    est_signal_two = generate_multivoxel_data(gtab, ivim_fit_two.model_params)

    assert_array_equal(est_signal_two.shape, data_two.shape)
    assert_array_almost_equal(est_signal_two, data_two)
    assert_array_almost_equal(ivim_fit_two.model_params, params)
<commit_msg>One stage fitting works with optimize<commit_after>""" Testing IVIM

"""
import numpy as np
from numpy.testing import (assert_array_equal, assert_array_almost_equal)

from dipy.reconst.ivim import ivim_function, IvimModel
from dipy.core.gradients import gradient_table
from dipy.sims.voxel import multi_tensor
from dipy.core.sphere import disperse_charges, HemiSphere

def test_fit_minimize():
    """
    Test the implementation of the fitting with minimize
    """
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)

    S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

    mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
    # This gives an isotropic signal

    signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
                          f * 100, 100 * (1 - f)])
    data = np.array([signal[0], ])

    ivim_model = IvimModel(gtab)
    ivim_fit = ivim_model.fit(data, routine="minimize")

    est_signal = np.array([ivim_function(ivim_fit.model_params[0], bvals), ])

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params[0], [S0, f, D_star, D])

def test_fit_leastsq():
    """
    Test the implementation of the fitting with leastsq
    """
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)

    S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

    mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
    # This gives an isotropic signal

    signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
                          f * 100, 100 * (1 - f)])
    data = signal[0]
    ivim_model = IvimModel(gtab)
    ivim_fit = ivim_model.fit(data, routine="leastsq")

    est_signal = ivim_function(ivim_fit.model_params, bvals)

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params, [S0, f, D_star, D])

def test_multivoxel():
    """Test fitting with multivoxel data"""
    bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
                      120., 140., 160., 180., 200., 220., 240.,
                      260., 280., 300., 350., 400., 500., 600.,
                      700., 800., 900., 1000.])
    N = len(bvals)
    bvecs = get_bvecs(N)
    gtab = gradient_table(bvals, bvecs.T)
    params = [[1.0, 0.2052, 0.00473, 0.00066], [1.0, 0.18, 0.00555, 0.0007]]

    data = generate_multivoxel_data(gtab, params)
    ivim_model = IvimModel(gtab)

    ivim_fit = ivim_model.fit(data)
    est_signal = generate_multivoxel_data(gtab, ivim_fit.model_params)

    assert_array_equal(est_signal.shape, data.shape)
    assert_array_almost_equal(est_signal, data)
    assert_array_almost_equal(ivim_fit.model_params, params)

def get_bvecs(N):
    """Generate bvectors for N bvalues"""
    theta = np.pi * np.random.rand(N)
    phi = 2 * np.pi * np.random.rand(N)
    hsph_initial = HemiSphere(theta=theta, phi=phi)
    hsph_updated, potential = disperse_charges(hsph_initial, 5000)
    vertices = hsph_updated.vertices
    return vertices

def generate_multivoxel_data(gtab, params):
    """Generate multivoxel data for testing"""
    data = []
    for parameters in params:
        S0, f, D_star, D = parameters
        signal = ivim_function(parameters, gtab.bvals)
        data.append(signal)
    return np.array(data)

# def test_two_stage():
#     """Test the two stage fitting routine"""
#     bvals = np.array([0., 10., 20., 30., 40., 60., 80., 100.,
#                       120., 140., 160., 180., 200., 220., 240.,
#                       260., 280., 300., 350., 400., 500., 600.,
#                       700., 800., 900., 1000.])
#     N = len(bvals)
#     bvecs = get_bvecs(N)
#     gtab = gradient_table(bvals, bvecs.T)

#     S0, f, D_star, D = 1.0, 0.2052, 0.00473, 0.00066

#     mevals = np.array(([D_star, D_star, D_star], [D, D, D]))
#     # This gives an isotropic signal

#     signal = multi_tensor(gtab, mevals, snr=None, S0=S0, fractions=[
#                           f * 100, 100 * (1 - f)])
#     data = signal[0]
#     ivim_model = IvimModel(gtab)

#     guess_params = np.array([[1.0, .10, 0.001, 0.0001], ])
#     ivim_fit = ivim_model.fit(data, fit_method="two_stage", routine='minimize')

#     est_signal = ivim_function(ivim_fit.model_params, bvals)

#     assert_array_equal(est_signal.shape, data.shape)
#     assert_array_almost_equal(est_signal, data)
#     assert_array_almost_equal(ivim_fit.model_params, [S0, f, D_star, D])

    # Test with multiple voxels

    # params = [[1.0, 0.2052, 0.00473, 0.00066], [1.0, 0.18, 0.00555, 0.0007]]

    # data_two = generate_multivoxel_data(gtab, params)
    # ivim_model_two = IvimModel(gtab)

    # guess_params = np.array([[1.0, 0.01, 0.001, 0.0009],
    #                          [0.9, 0.04, 0.002, 0.0004]])
    # ivim_fit_two = ivim_model_two.fit(data_two, fit_method="two_stage",
    #                                   x0=guess_params)
    # est_signal_two = generate_multivoxel_data(gtab, ivim_fit_two.model_params)

    # assert_array_equal(est_signal_two.shape, data_two.shape)
    # assert_array_almost_equal(est_signal_two, data_two)
    # assert_array_almost_equal(ivim_fit_two.model_params, params)
<|endoftext|>


<commit_before>import django_rq
from django.core.cache import cache
from mrbelvedereci.cumulusci.models import Org
from mrbelvedereci.repository.utils import create_status

@django_rq.job('default', timeout=14400)
def run_build(build_id, lock_id=None):
    from mrbelvedereci.build.models import Build
    try:
        build = Build.objects.get(id=build_id)
        exception = None
        build.run()
        res_status = set_github_status.delay(build_id)
        build.task_id_status_end = res_status.id
        build.save()

    except Exception as e:
        if lock_id:
            cache.delete(lock_id)

        res_status = set_github_status.delay(build_id)
        build.task_id_status_end = res_status.id
        build.status = 'error'
        build.log += '\nERROR: The build raised an exception\n'
        build.log += unicode(e)
        build.save()

    if lock_id:
        cache.delete(lock_id)

    return build.status

@django_rq.job('short', timeout=60)
def check_queued_build(build_id):
    from mrbelvedereci.build.models import Build
    try:
        build = Build.objects.get(id = build_id)
    except Build.DoesNotExist:
        time.sleep(1)
        check_queued_build.delay(build_id)

    # Check for concurrency blocking
    try:
        org = Org.objects.get(name = build.plan.org, repo = build.repo)
    except Org.DoesNotExist:
        return

    if org.scratch:
        # For scratch orgs, we don't need concurrency blocking logic, just run the build
        res_run = run_build.delay(build.id)
        build.task_id_run = res_run.id

    else:
        # For persistent orgs, use the cache to lock the org
        lock_id = 'mrbelvedereci-org-lock-{}'.format(org.id)
        status = cache.add(lock_id, 'build-{}'.format(build_id))

        if status is True:
            # Lock successful, run the build
            res_run = run_build.delay(build.id, lock_id)
            build.task_id_run = res_run.id
            build.save()
        else:
            # Failed to get lock, queue next check
            run_check = check_queued_build.delay(build.id)
            build.task_id_check = res_check.id
            build.save()

@django_rq.job('short')
def set_github_status(build_id):
    from mrbelvedereci.build.models import Build
    build = Build.objects.get(id = build_id)
    create_status(build)

@django_rq.job('short')
def check_build_tasks():
    from mrbelvedereci.build.models import Build
    builds = Build.objects.filter(status = 'running')
    for build in builds:
        task_id = build.task_id_run
<commit_msg>Fix variable name error in concurrency logic<commit_after>import django_rq
from django.core.cache import cache
from mrbelvedereci.cumulusci.models import Org
from mrbelvedereci.repository.utils import create_status

@django_rq.job('default', timeout=14400)
def run_build(build_id, lock_id=None):
    from mrbelvedereci.build.models import Build
    try:
        build = Build.objects.get(id=build_id)
        exception = None
        build.run()
        res_status = set_github_status.delay(build_id)
        build.task_id_status_end = res_status.id
        build.save()

    except Exception as e:
        if lock_id:
            cache.delete(lock_id)

        res_status = set_github_status.delay(build_id)
        build.task_id_status_end = res_status.id
        build.status = 'error'
        build.log += '\nERROR: The build raised an exception\n'
        build.log += unicode(e)
        build.save()

    if lock_id:
        cache.delete(lock_id)

    return build.status

@django_rq.job('short', timeout=60)
def check_queued_build(build_id):
    from mrbelvedereci.build.models import Build
    try:
        build = Build.objects.get(id = build_id)
    except Build.DoesNotExist:
        time.sleep(1)
        check_queued_build.delay(build_id)

    # Check for concurrency blocking
    try:
        org = Org.objects.get(name = build.plan.org, repo = build.repo)
    except Org.DoesNotExist:
        return

    if org.scratch:
        # For scratch orgs, we don't need concurrency blocking logic, just run the build
        res_run = run_build.delay(build.id)
        build.task_id_run = res_run.id

    else:
        # For persistent orgs, use the cache to lock the org
        lock_id = 'mrbelvedereci-org-lock-{}'.format(org.id)
        status = cache.add(lock_id, 'build-{}'.format(build_id))

        if status is True:
            # Lock successful, run the build
            res_run = run_build.delay(build.id, lock_id)
            build.task_id_run = res_run.id
            build.save()
        else:
            # Failed to get lock, queue next check
            res_check = check_queued_build.delay(build.id)
            build.task_id_check = res_check.id
            build.save()

@django_rq.job('short')
def set_github_status(build_id):
    from mrbelvedereci.build.models import Build
    build = Build.objects.get(id = build_id)
    create_status(build)

@django_rq.job('short')
def check_build_tasks():
    from mrbelvedereci.build.models import Build
    builds = Build.objects.filter(status = 'running')
    for build in builds:
        task_id = build.task_id_run
<|endoftext|>


<commit_before># Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import io
import csv

from taiga.base.utils import db, text
from taiga.projects.history.services import take_snapshot
from taiga.projects.tasks.apps import (
    connect_tasks_signals,
    disconnect_tasks_signals)
from taiga.events import events
from taiga.projects.votes.utils import attach_total_voters_to_queryset
from taiga.projects.notifications.utils import attach_watchers_to_queryset

from . import models

def get_tasks_from_bulk(bulk_data, **additional_fields):
    """Convert `bulk_data` into a list of tasks.

    :param bulk_data: List of tasks in bulk format.
    :param additional_fields: Additional fields when instantiating each task.

    :return: List of `Task` instances.
    """
    return [models.Task(subject=line, **additional_fields)
            for line in text.split_in_lines(bulk_data)]

def create_tasks_in_bulk(bulk_data, callback=None, precall=None, **additional_fields):
    """Create tasks from `bulk_data`.

    :param bulk_data: List of tasks in bulk format.
    :param callback: Callback to execute after each task save.
    :param additional_fields: Additional fields when instantiating each task.

    :return: List of created `Task` instances.
    """
    tasks = get_tasks_from_bulk(bulk_data, **additional_fields)

    disconnect_tasks_signals()

    try:
        db.save_in_bulk(tasks, callback, precall)
    finally:
        connect_tasks_signals()

    return tasks

def update_tasks_order_in_bulk(bulk_data:list, field:str, project:object):
    """
    Update the order of some tasks.
    `bulk_data` should be a list of tuples with the following format:

    [(<task id>, {<field>: <value>, ...}), ...]
    """
    task_ids = []
    new_order_values = []
    for task_data in bulk_data:
        task_ids.append(task_data["task_id"])
        new_order_values.append({field: task_data["order"]})

    events.emit_event_for_ids(ids=task_ids,
                              content_type="tasks.task",
                              projectid=project.pk)

    db.update_in_bulk_with_ids(task_ids, new_order_values, model=models.Task)

def snapshot_tasks_in_bulk(bulk_data, user):
    task_ids = []
    for task_data in bulk_data:
        try:
            task = models.Task.objects.get(pk=task_data['task_id'])
            take_snapshot(task, user=user)
        except models.UserStory.DoesNotExist:
            pass

def tasks_to_csv(project, queryset):
    csv_data = io.StringIO()
    fieldnames = ["ref", "subject", "description", "user_story", "sprint", "sprint_estimated_start",
                  "sprint_estimated_finish", "owner", "owner_full_name", "assigned_to",
                  "assigned_to_full_name", "status", "is_iocaine", "is_closed", "us_order",
                  "taskboard_order", "attachments", "external_reference", "tags", "watchers", "voters"]

    custom_attrs = project.taskcustomattributes.all()
    for custom_attr in custom_attrs:
        fieldnames.append(custom_attr.name)

    queryset = queryset.prefetch_related("attachments",
                                         "custom_attributes_values")
    queryset = queryset.select_related("milestone",
                                       "owner",
                                       "assigned_to",
                                       "status",
                                       "project")

    queryset = attach_total_voters_to_queryset(queryset)
    queryset = attach_watchers_to_queryset(queryset)

    writer = csv.DictWriter(csv_data, fieldnames=fieldnames)
    writer.writeheader()
    for task in queryset:
        task_data = {
            "ref": task.ref,
            "subject": task.subject,
            "description": task.description,
            "user_story": task.user_story.ref if task.user_story else None,
            "sprint": task.milestone.name if task.milestone else None,
            "sprint_estimated_start": task.milestone.estimated_start if task.milestone else None,
            "sprint_estimated_finish": task.milestone.estimated_finish if task.milestone else None,
            "owner": task.owner.username if task.owner else None,
            "owner_full_name": task.owner.get_full_name() if task.owner else None,
            "assigned_to": task.assigned_to.username if task.assigned_to else None,
            "assigned_to_full_name": task.assigned_to.get_full_name() if task.assigned_to else None,
            "status": task.status.name if task.status else None,
            "is_iocaine": task.is_iocaine,
            "is_closed": task.status is not None and task.status.is_closed,
            "us_order": task.us_order,
            "taskboard_order": task.taskboard_order,
            "attachments": task.attachments.count(),
            "external_reference": task.external_reference,
            "tags": ",".join(task.tags or []),
            "watchers": task.watchers,
            "voters": task.total_voters,
        }
        for custom_attr in custom_attrs:
            value = task.custom_attributes_values.attributes_values.get(str(custom_attr.id), None)
            task_data[custom_attr.name] = value

        writer.writerow(task_data)

    return csv_data
<commit_msg>[Backport] Fix error, import a bad model class<commit_after># Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import io
import csv

from taiga.base.utils import db, text
from taiga.projects.history.services import take_snapshot
from taiga.projects.tasks.apps import (
    connect_tasks_signals,
    disconnect_tasks_signals)
from taiga.events import events
from taiga.projects.votes.utils import attach_total_voters_to_queryset
from taiga.projects.notifications.utils import attach_watchers_to_queryset

from . import models

def get_tasks_from_bulk(bulk_data, **additional_fields):
    """Convert `bulk_data` into a list of tasks.

    :param bulk_data: List of tasks in bulk format.
    :param additional_fields: Additional fields when instantiating each task.

    :return: List of `Task` instances.
    """
    return [models.Task(subject=line, **additional_fields)
            for line in text.split_in_lines(bulk_data)]

def create_tasks_in_bulk(bulk_data, callback=None, precall=None, **additional_fields):
    """Create tasks from `bulk_data`.

    :param bulk_data: List of tasks in bulk format.
    :param callback: Callback to execute after each task save.
    :param additional_fields: Additional fields when instantiating each task.

    :return: List of created `Task` instances.
    """
    tasks = get_tasks_from_bulk(bulk_data, **additional_fields)

    disconnect_tasks_signals()

    try:
        db.save_in_bulk(tasks, callback, precall)
    finally:
        connect_tasks_signals()

    return tasks

def update_tasks_order_in_bulk(bulk_data:list, field:str, project:object):
    """
    Update the order of some tasks.
    `bulk_data` should be a list of tuples with the following format:

    [(<task id>, {<field>: <value>, ...}), ...]
    """
    task_ids = []
    new_order_values = []
    for task_data in bulk_data:
        task_ids.append(task_data["task_id"])
        new_order_values.append({field: task_data["order"]})

    events.emit_event_for_ids(ids=task_ids,
                              content_type="tasks.task",
                              projectid=project.pk)

    db.update_in_bulk_with_ids(task_ids, new_order_values, model=models.Task)

def snapshot_tasks_in_bulk(bulk_data, user):
    task_ids = []
    for task_data in bulk_data:
        try:
            task = models.Task.objects.get(pk=task_data['task_id'])
            take_snapshot(task, user=user)
        except models.Task.DoesNotExist:
            pass

def tasks_to_csv(project, queryset):
    csv_data = io.StringIO()
    fieldnames = ["ref", "subject", "description", "user_story", "sprint", "sprint_estimated_start",
                  "sprint_estimated_finish", "owner", "owner_full_name", "assigned_to",
                  "assigned_to_full_name", "status", "is_iocaine", "is_closed", "us_order",
                  "taskboard_order", "attachments", "external_reference", "tags", "watchers", "voters"]

    custom_attrs = project.taskcustomattributes.all()
    for custom_attr in custom_attrs:
        fieldnames.append(custom_attr.name)

    queryset = queryset.prefetch_related("attachments",
                                         "custom_attributes_values")
    queryset = queryset.select_related("milestone",
                                       "owner",
                                       "assigned_to",
                                       "status",
                                       "project")

    queryset = attach_total_voters_to_queryset(queryset)
    queryset = attach_watchers_to_queryset(queryset)

    writer = csv.DictWriter(csv_data, fieldnames=fieldnames)
    writer.writeheader()
    for task in queryset:
        task_data = {
            "ref": task.ref,
            "subject": task.subject,
            "description": task.description,
            "user_story": task.user_story.ref if task.user_story else None,
            "sprint": task.milestone.name if task.milestone else None,
            "sprint_estimated_start": task.milestone.estimated_start if task.milestone else None,
            "sprint_estimated_finish": task.milestone.estimated_finish if task.milestone else None,
            "owner": task.owner.username if task.owner else None,
            "owner_full_name": task.owner.get_full_name() if task.owner else None,
            "assigned_to": task.assigned_to.username if task.assigned_to else None,
            "assigned_to_full_name": task.assigned_to.get_full_name() if task.assigned_to else None,
            "status": task.status.name if task.status else None,
            "is_iocaine": task.is_iocaine,
            "is_closed": task.status is not None and task.status.is_closed,
            "us_order": task.us_order,
            "taskboard_order": task.taskboard_order,
            "attachments": task.attachments.count(),
            "external_reference": task.external_reference,
            "tags": ",".join(task.tags or []),
            "watchers": task.watchers,
            "voters": task.total_voters,
        }
        for custom_attr in custom_attrs:
            value = task.custom_attributes_values.attributes_values.get(str(custom_attr.id), None)
            task_data[custom_attr.name] = value

        writer.writerow(task_data)

    return csv_data
<|endoftext|>


<commit_before>from DirectFrame import *

"""
import DirectWaitBar
d = DirectWaitBar(borderWidth=(0,0))

"""

class DirectWaitBar(DirectFrame):
    """
    DirectEntry(parent) - Create a DirectGuiWidget which responds
    to keyboard buttons
    """
    def __init__(self, parent = aspect2d, **kw):
        # Inherits from DirectFrame
        # A Direct Frame can have:
        # - A background texture (pass in path to image, or Texture Card)
        # - A midground geometry item (pass in geometry)
        # - A foreground text Node (pass in text string or Onscreen Text)
        optiondefs = (
            # Define type of DirectGuiWidget
            ('pgFunc',         PGWaitBar,          None),
            ('frameSize',      (-1,1,-0.08,0.08),  None),
            ('borderWidth',    (0,0),              None),
            ('range',          100,                self.setRange),
            ('value',          50,                 self.setValue),
            ('barBorderWidth', (0,0),              self.setBarBorderWidth),
            ('barColor',       (1,0,0,1),          self.setBarColor),
            ('barRelief',      FLAT,               self.setBarRelief),
            ('sortOrder',      NO_FADE_SORT_INDEX, None),
            )
        if kw.has_key('text'):
            textoptiondefs = (
                ('text_pos',    (0,-0.025),          None),
                ('text_scale',  0.1,                 None)
                )
        else:
            textoptiondefs = ()
        # Merge keyword options with default options
        self.defineoptions(kw, optiondefs + textoptiondefs)
        # Initialize superclasses
        DirectFrame.__init__(self, parent)
        self.barStyle = PGFrameStyle()
        # Call option initialization functions
        self.initialiseoptions(DirectWaitBar)
        self.updateBarStyle()

    def destroy(self):
        del self.barStyle
        DirectFrame.destroy(self)

    def setRange(self):
        self.guiItem.setRange(self['range'])

    def setValue(self):
        self.guiItem.setValue(self['value'])

    def getPercent(self):
        return self.guiItem.getPercent()

    def updateBarStyle(self):
        if not self.fInit:
            self.guiItem.setBarStyle(self.barStyle)

    def setBarRelief(self):
        self.barStyle.setType(self['barRelief'])
        self.updateBarStyle()

    def setBarBorderWidth(self):
        self.barStyle.setWidth(*self['barBorderWidth'])
        self.updateBarStyle()

    def setBarColor(self):
        color = self['barColor']
        self.barStyle.setColor(color[0], color[1], color[2], color[3])
        self.updateBarStyle()

    def update(self, value):
        self['value'] = value

        # Render a frame out-of-sync with the igloop to update the
        # window right now.  This allows the wait bar to be updated
        # even though we are not normally rendering frames.
        base.graphicsEngine.renderFrame()

    def finish(self):
        # Fill the bar in N frames
        N = 10 # take 10 frames
        remaining = self['range'] - self['value']
        if remaining:
            step = max(1, int(remaining / N))
            count = self['value']
            while count != self['range']:
                count += step
                if count > self['range']:
                    count = self['range']
                self.update(count)

<commit_msg>start wait bars at 0<commit_after>from DirectFrame import *

"""
import DirectWaitBar
d = DirectWaitBar(borderWidth=(0,0))

"""

class DirectWaitBar(DirectFrame):
    """
    DirectEntry(parent) - Create a DirectGuiWidget which responds
    to keyboard buttons
    """
    def __init__(self, parent = aspect2d, **kw):
        # Inherits from DirectFrame
        # A Direct Frame can have:
        # - A background texture (pass in path to image, or Texture Card)
        # - A midground geometry item (pass in geometry)
        # - A foreground text Node (pass in text string or Onscreen Text)
        optiondefs = (
            # Define type of DirectGuiWidget
            ('pgFunc',         PGWaitBar,          None),
            ('frameSize',      (-1,1,-0.08,0.08),  None),
            ('borderWidth',    (0,0),              None),
            ('range',          100,                self.setRange),
            ('value',          0,                  self.setValue),
            ('barBorderWidth', (0,0),              self.setBarBorderWidth),
            ('barColor',       (1,0,0,1),          self.setBarColor),
            ('barRelief',      FLAT,               self.setBarRelief),
            ('sortOrder',      NO_FADE_SORT_INDEX, None),
            )
        if kw.has_key('text'):
            textoptiondefs = (
                ('text_pos',    (0,-0.025),          None),
                ('text_scale',  0.1,                 None)
                )
        else:
            textoptiondefs = ()
        # Merge keyword options with default options
        self.defineoptions(kw, optiondefs + textoptiondefs)
        # Initialize superclasses
        DirectFrame.__init__(self, parent)
        self.barStyle = PGFrameStyle()
        # Call option initialization functions
        self.initialiseoptions(DirectWaitBar)
        self.updateBarStyle()

    def destroy(self):
        del self.barStyle
        DirectFrame.destroy(self)

    def setRange(self):
        self.guiItem.setRange(self['range'])

    def setValue(self):
        self.guiItem.setValue(self['value'])

    def getPercent(self):
        return self.guiItem.getPercent()

    def updateBarStyle(self):
        if not self.fInit:
            self.guiItem.setBarStyle(self.barStyle)

    def setBarRelief(self):
        self.barStyle.setType(self['barRelief'])
        self.updateBarStyle()

    def setBarBorderWidth(self):
        self.barStyle.setWidth(*self['barBorderWidth'])
        self.updateBarStyle()

    def setBarColor(self):
        color = self['barColor']
        self.barStyle.setColor(color[0], color[1], color[2], color[3])
        self.updateBarStyle()

    def update(self, value):
        self['value'] = value

        # Render a frame out-of-sync with the igloop to update the
        # window right now.  This allows the wait bar to be updated
        # even though we are not normally rendering frames.
        base.graphicsEngine.renderFrame()

    def finish(self):
        # Fill the bar in N frames
        N = 10 # take 10 frames
        remaining = self['range'] - self['value']
        if remaining:
            step = max(1, int(remaining / N))
            count = self['value']
            while count != self['range']:
                count += step
                if count > self['range']:
                    count = self['range']
                self.update(count)

<|endoftext|>


<commit_before>"""
Functionalities for reading and writing Higgins Annotation Tool (HAT) XML annotation files <http://www.speech.kth.se/hat/>.
"""

__author__ = "Todd Shore <errantlinguist+github@gmail.com>"
__copyright__ = "Copyright (C) 2016-2017 Todd Shore"
__license__ = "Apache License, Version 2.0"

import re
from collections import defaultdict
from typing import Mapping, Optional, Tuple, Union
from xml.sax.saxutils import escape

import lxml.builder
import lxml.etree as etree

HAT_DATA_NAMESPACE = "http://www.speech.kth.se/higgins/2005/annotation/"
HAT_DATA_NAMESPACE_NAME = "hat"
HAT_DATA_SCHEMA_LOCATION = "http://www.speech.kth.se/higgins/2005/annotation/annotation.xsd"
ANNOTATION_NAMESPACES = {HAT_DATA_NAMESPACE_NAME: HAT_DATA_NAMESPACE,
						 "xsi": "http://www.w3.org/2001/XMLSchema-instance"}

ELEMENT_MAKER = lxml.builder.ElementMaker(
	nsmap=ANNOTATION_NAMESPACES)

__DIGITS_PATTERN = re.compile('(\d+)')
__WHITESPACE_PATTERN = re.compile('\s+')

class AnnotationData(object):
	def __init__(self, qname_factory, element_maker: lxml.builder.ElementMaker, encoding: str):
		self.__qname_factory = qname_factory
		self.element_maker = element_maker
		self.encoding = encoding
		self.track_data = {}
		self.segment_data = SegmentData(qname_factory)

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, other):
		old_track_count = len(self.track_data)
		for track_id, other_track in other.track_data.items():
			if track_id in self.track_data:
				raise ValueError("Track ID \"%s\" already in dict." % track_id)
			else:
				self.track_data[track_id] = other_track
		# Remove channel IDs because HAT doesn't support multiple tracks for one Annotation which each have "channel" attrs
		if old_track_count < len(self.track_data):
			for track_datum in self.track_data.values():
				track_datum.remove_attr("channel")

		self.segment_data.add(other.segment_data)

	def create_xml_element(self):
		# http://stackoverflow.com/a/22902367/1391325
		result = self.element_maker("annotation")
		tracks_elem = etree.SubElement(result, self.__qname_factory("tracks"))
		for track_id, track_datum in sorted(self.track_data.items(), key=lambda k: natural_keys(k[0])):
			track_elem = etree.SubElement(tracks_elem, self.__qname_factory("track"), attrib={"id": track_id})
			sources_elem = etree.SubElement(track_elem, self.__qname_factory("sources"))
			for track_source in sorted(track_datum.sources_by_id.values(),
									   key=lambda elem: natural_keys(elem.get("id"))):
				sources_elem.append(track_source)

		segments_elem = self.segment_data.create_xml_element()
		result.append(segments_elem)

		return result

class AnnotationParser(object):
	def __init__(self, qname_factory, nsmap: Optional[Mapping[Optional[str], str]] = None, id_prefix: str = ""):
		self.__qname_factory = qname_factory
		self.element_maker = ELEMENT_MAKER if nsmap is None else lxml.builder.ElementMaker(nsmap)
		self.id_prefix = id_prefix
		self.__tag_parsers = {self.__qname_factory("tracks"): self.__parse_tracks,
							  self.__qname_factory("segments"): self.__parse_segments}

	def __call__(self, doc_tree):
		result = AnnotationData(self.__qname_factory, self.element_maker, doc_tree.docinfo.encoding)
		tag_name = self.__qname_factory("annotation")
		for child in doc_tree.iter(tag_name):
			self.__parse_annotation(child, result)

		return result

	def __parse_annotation(self, annot, result):
		for child in annot:
			tag_name = child.tag
			parser = self.__tag_parsers[tag_name]
			parser(child, result)

	def __parse_segments(self, segments, result):
		segment_data = result.segment_data
		for segment in segments:
			attrs = segment.attrib
			segment_id = self.id_prefix + attrs["id"]
			attrs["id"] = segment_id
			segment_data.segments_by_id[segment_id] = segment
			track_id = self.id_prefix + attrs["track"]
			segment_data.track_segments[track_id].append(segment)
			attrs["track"] = track_id
			source_id = self.id_prefix + attrs["source"]
			segment_data.source_segments[source_id].append(segment)
			attrs["source"] = source_id

	def __parse_tracks(self, tracks, result):
		source_tag_name = self.__qname_factory("source")
		track_data = result.track_data
		for track in track_data:
			track_source_data = TrackDatum()
			track_id = self.id_prefix + track.get("id")
			track.set("id", track_id)
			track_data[track_id] = track_source_data

			for source in track.iter(source_tag_name):
				source_id = self.id_prefix + source.get("id")
				source.set("id", source_id)
				track_source_data.add(source)

class QNameStringFactory(object):
	def __init__(self, namespace):
		self.tag_qnames = {}
		self.__tag_prefix = "{" + namespace + "}"

	def __call__(self, tag_name: str):
		result = self.tag_qnames.get(tag_name)
		if not result:
			result = self.__tag_prefix + tag_name
			self.tag_qnames[tag_name] = result
		return result

class SegmentData(object):
	def __init__(self, qname_factory):
		self.__qname_factory = qname_factory
		self.segments_by_id = {}
		self.track_segments = defaultdict(list)
		self.source_segments = defaultdict(list)

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, other):
		for segment_id, segment in other.segments_by_id.items():
			if segment_id in self.segments_by_id:
				raise ValueError("Segment ID \"%s\" already in dict." % segment_id)
			else:
				self.segments_by_id[segment_id] = segment
				self.track_segments.update(other.track_segments)
				self.source_segments.update(other.source_segments)

	def create_xml_element(self):
		result = etree.Element(self.__qname_factory("segments"))
		for segment in sorted(self.segments_by_id.values(), key=lambda elem: natural_keys(elem.get("id"))):
			result.append(segment)

		return result

class TrackDatum(object):
	def __init__(self):
		self.sources_by_id = {}
		self.sources_by_channel = {}
		self.sources_by_href = {}

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, source):
		source_id = source.get("id")
		self.sources_by_id[source_id] = source
		channel = source.get("channel")
		if not is_blank_or_none(channel):
			self.sources_by_channel[channel] = source
		href = source.get("href")
		if not is_blank_or_none(href):
			self.sources_by_href[href] = source

	def remove(self, source):
		source_id = source.get("id")
		del self.sources_by_id[source_id]
		channel = source.get("channel")
		if not is_blank_or_none(channel):
			del self.sources_by_channel[channel]
		href = source.get("href")
		if not is_blank_or_none(href):
			del self.sources_by_href[href]

	def remove_attr(self, attr):
		if attr == "id":
			raise ValueError("Cannot remove ID attribute: This is absolutely necessary.")
		else:
			for source in self.sources_by_id.values():
				# source.set(attr, None)
				try:
					del source.attrib["channel"]
				except KeyError:
					# Attr not present; just continue
					pass

def is_blank_or_none(string: str) -> bool:
	return string is None or len(string) < 1 or string.isspace()

def natural_keys(text: str) -> Tuple[Union[int, str], ...]:
	"""
	alist.sort(key=natural_keys) sorts in human order

	:see: http://nedbatchelder.com/blog/200712/human_sorting.html
	:see: http://stackoverflow.com/a/5967539/1391325
	"""
	return tuple(__atoi(c) for c in __DIGITS_PATTERN.split(text))

def sanitize_dom_id(string: str) -> str:
	result = __WHITESPACE_PATTERN.sub('-', string)
	return escape(result)

def __atoi(text: str) -> Union[int, str]:
	"""
	:see: http://stackoverflow.com/a/5967539/1391325
	"""
	return int(text) if text.isdigit() else text

def __create_track_source_elem(tracks_elem, source_id: str, channel: str, audio_file_path: str):
	track_sources = ELEMENT_MAKER.SubElement(tracks_elem, "hat:sources")
	result = ELEMENT_MAKER.SubElement(track_sources, "hat:source")
	result.set("id", source_id)
	result.set("channel", channel)
	result.set("href", audio_file_path)
	return result
<commit_msg>added type hints<commit_after>"""
Functionalities for reading and writing Higgins Annotation Tool (HAT) XML annotation files <http://www.speech.kth.se/hat/>.
"""

__author__ = "Todd Shore <errantlinguist+github@gmail.com>"
__copyright__ = "Copyright (C) 2016-2017 Todd Shore"
__license__ = "Apache License, Version 2.0"

import re
from collections import defaultdict
from typing import Mapping, Optional, Tuple, Union
from xml.sax.saxutils import escape

import lxml.builder
import lxml.etree as etree

HAT_DATA_NAMESPACE = "http://www.speech.kth.se/higgins/2005/annotation/"
HAT_DATA_NAMESPACE_NAME = "hat"
HAT_DATA_SCHEMA_LOCATION = "http://www.speech.kth.se/higgins/2005/annotation/annotation.xsd"
ANNOTATION_NAMESPACES = {HAT_DATA_NAMESPACE_NAME: HAT_DATA_NAMESPACE,
						 "xsi": "http://www.w3.org/2001/XMLSchema-instance"}

ELEMENT_MAKER = lxml.builder.ElementMaker(
	nsmap=ANNOTATION_NAMESPACES)

__DIGITS_PATTERN = re.compile('(\d+)')
__WHITESPACE_PATTERN = re.compile('\s+')

class AnnotationData(object):
	def __init__(self, qname_factory, element_maker: lxml.builder.ElementMaker, encoding: str):
		self.__qname_factory = qname_factory
		self.element_maker = element_maker
		self.encoding = encoding
		self.track_data = {}
		self.segment_data = SegmentData(qname_factory)

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, other):
		old_track_count = len(self.track_data)
		for track_id, other_track in other.track_data.items():
			if track_id in self.track_data:
				raise ValueError("Track ID \"%s\" already in dict." % track_id)
			else:
				self.track_data[track_id] = other_track
		# Remove channel IDs because HAT doesn't support multiple tracks for one Annotation which each have "channel" attrs
		if old_track_count < len(self.track_data):
			for track_datum in self.track_data.values():
				track_datum.remove_attr("channel")

		self.segment_data.add(other.segment_data)

	def create_xml_element(self) -> etree.Element:
		# http://stackoverflow.com/a/22902367/1391325
		result = self.element_maker("annotation")
		tracks_elem = etree.SubElement(result, self.__qname_factory("tracks"))
		for track_id, track_datum in sorted(self.track_data.items(), key=lambda k: natural_keys(k[0])):
			track_elem = etree.SubElement(tracks_elem, self.__qname_factory("track"), attrib={"id": track_id})
			sources_elem = etree.SubElement(track_elem, self.__qname_factory("sources"))
			for track_source in sorted(track_datum.sources_by_id.values(),
									   key=lambda elem: natural_keys(elem.get("id"))):
				sources_elem.append(track_source)

		segments_elem = self.segment_data.create_xml_element()
		result.append(segments_elem)

		return result

class AnnotationParser(object):
	def __init__(self, qname_factory, nsmap: Optional[Mapping[Optional[str], str]] = None, id_prefix: str = ""):
		self.__qname_factory = qname_factory
		self.element_maker = ELEMENT_MAKER if nsmap is None else lxml.builder.ElementMaker(nsmap)
		self.id_prefix = id_prefix
		self.__tag_parsers = {self.__qname_factory("tracks"): self.__parse_tracks,
							  self.__qname_factory("segments"): self.__parse_segments}

	def __call__(self, doc_tree) -> AnnotationData:
		result = AnnotationData(self.__qname_factory, self.element_maker, doc_tree.docinfo.encoding)
		tag_name = self.__qname_factory("annotation")
		for child in doc_tree.iter(tag_name):
			self.__parse_annotation(child, result)

		return result

	def __parse_annotation(self, annot, result):
		for child in annot:
			tag_name = child.tag
			parser = self.__tag_parsers[tag_name]
			parser(child, result)

	def __parse_segments(self, segments, result):
		segment_data = result.segment_data
		for segment in segments:
			attrs = segment.attrib
			segment_id = self.id_prefix + attrs["id"]
			attrs["id"] = segment_id
			segment_data.segments_by_id[segment_id] = segment
			track_id = self.id_prefix + attrs["track"]
			segment_data.track_segments[track_id].append(segment)
			attrs["track"] = track_id
			source_id = self.id_prefix + attrs["source"]
			segment_data.source_segments[source_id].append(segment)
			attrs["source"] = source_id

	def __parse_tracks(self, tracks, result):
		source_tag_name = self.__qname_factory("source")
		track_data = result.track_data
		for track in track_data:
			track_source_data = TrackDatum()
			track_id = self.id_prefix + track.get("id")
			track.set("id", track_id)
			track_data[track_id] = track_source_data

			for source in track.iter(source_tag_name):
				source_id = self.id_prefix + source.get("id")
				source.set("id", source_id)
				track_source_data.add(source)

class QNameStringFactory(object):
	def __init__(self, namespace):
		self.tag_qnames = {}
		self.__tag_prefix = "{" + namespace + "}"

	def __call__(self, tag_name: str):
		result = self.tag_qnames.get(tag_name)
		if not result:
			result = self.__tag_prefix + tag_name
			self.tag_qnames[tag_name] = result
		return result

class SegmentData(object):
	def __init__(self, qname_factory):
		self.__qname_factory = qname_factory
		self.segments_by_id = {}
		self.track_segments = defaultdict(list)
		self.source_segments = defaultdict(list)

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, other):
		for segment_id, segment in other.segments_by_id.items():
			if segment_id in self.segments_by_id:
				raise ValueError("Segment ID \"%s\" already in dict." % segment_id)
			else:
				self.segments_by_id[segment_id] = segment
				self.track_segments.update(other.track_segments)
				self.source_segments.update(other.source_segments)

	def create_xml_element(self) -> etree.Element:
		result = etree.Element(self.__qname_factory("segments"))
		for segment in sorted(self.segments_by_id.values(), key=lambda elem: natural_keys(elem.get("id"))):
			result.append(segment)

		return result

class TrackDatum(object):
	def __init__(self):
		self.sources_by_id = {}
		self.sources_by_channel = {}
		self.sources_by_href = {}

	def __repr__(self):
		return self.__class__.__name__ + str(self.__dict__)

	def add(self, source):
		source_id = source.get("id")
		self.sources_by_id[source_id] = source
		channel = source.get("channel")
		if not is_blank_or_none(channel):
			self.sources_by_channel[channel] = source
		href = source.get("href")
		if not is_blank_or_none(href):
			self.sources_by_href[href] = source

	def remove(self, source):
		source_id = source.get("id")
		del self.sources_by_id[source_id]
		channel = source.get("channel")
		if not is_blank_or_none(channel):
			del self.sources_by_channel[channel]
		href = source.get("href")
		if not is_blank_or_none(href):
			del self.sources_by_href[href]

	def remove_attr(self, attr: str):
		if attr == "id":
			raise ValueError("Cannot remove ID attribute: This is absolutely necessary.")
		else:
			for source in self.sources_by_id.values():
				# source.set(attr, None)
				try:
					del source.attrib["channel"]
				except KeyError:
					# Attr not present; just continue
					pass

def is_blank_or_none(string: str) -> bool:
	return string is None or len(string) < 1 or string.isspace()

def natural_keys(text: str) -> Tuple[Union[int, str], ...]:
	"""
	alist.sort(key=natural_keys) sorts in human order

	:see: http://nedbatchelder.com/blog/200712/human_sorting.html
	:see: http://stackoverflow.com/a/5967539/1391325
	"""
	return tuple(__atoi(c) for c in __DIGITS_PATTERN.split(text))

def sanitize_dom_id(string: str) -> str:
	result = __WHITESPACE_PATTERN.sub('-', string)
	return escape(result)

def __atoi(text: str) -> Union[int, str]:
	"""
	:see: http://stackoverflow.com/a/5967539/1391325
	"""
	return int(text) if text.isdigit() else text

def __create_track_source_elem(tracks_elem, source_id: str, channel: str, audio_file_path: str):
	track_sources = ELEMENT_MAKER.SubElement(tracks_elem, "hat:sources")
	result = ELEMENT_MAKER.SubElement(track_sources, "hat:source")
	result.set("id", source_id)
	result.set("channel", channel)
	result.set("href", audio_file_path)
	return result
<|endoftext|>


<commit_before>"""
LogicGateAI.py

    Logic Gates:

         and: 0 0 = 0     or: 0 0 = 0    xor: 0 0 = 0
              0 1 = 0         0 1 = 1         0 1 = 1
              1 0 = 0         1 0 = 1         1 0 = 1
              1 1 = 1         1 1 = 1         1 1 = 0

        nand: 0 0 = 1    nor: 0 0 = 1   xnor: 0 0 = 1
              0 1 = 1         0 1 = 0         0 1 = 0
              1 0 = 1         1 0 = 0         1 0 = 0
              1 1 = 0         1 1 = 0         1 1 = 1

    In the following:
        1: send a true message
        0: send a false message
        -: don't send a message

        a b  and  or  xor  nand  nor  xnor
       (0 0)  (0) (0)  (0)   (1)  (1)   (1)  <--- initial state
        1 0    -   1    1     -    0     0
        0 0    -   0    0     -    1     1
        1 0    -   1    1     -    0     0
        1 1    1   -    0     0    -     1
        0 1    0   -    1     1    -     0
        1 1    1   -    0     0    -     1
        0 1    0   -    1     1    -     0
        0 0    -   0    0     -    1     1
"""

import PandaObject
import DirectNotifyGlobal
import Entity

def andTest(self, a, b):
    assert(self.debugPrint("andTest(a=%s, b=%s)"%(a, b)))
    if b:
        messenger.send(self.getName(), [a])

def orTest(self, a, b):
    assert(self.debugPrint("orTest(a=%s, b=%s)"%(a, b)))
    if not b:
        messenger.send(self.getName(), [a])
    # else: ...we already sent the messege when b was set.

def xorTest(self, a, b):
    assert(self.debugPrint("xorTest(a=%s, b=%s)"%(a, b)))
    messenger.send(self.getName(), [(not (a and b)) and (a or b)])

def nandTest(self, a, b):
    assert(self.debugPrint("nandTest(a=%s, b=%s)"%(a, b)))
    if b:
        messenger.send(self.getName(), [not (a and b)])

def norTest(self, a, b):
    assert(self.debugPrint("norTest(a=%s, b=%s)"%(a, b)))
    if not b:
        messenger.send(self.getName(), [not (a or b)])
    # else: ...we already sent the messege when b was set.

def xnorTest(self, a, b):
    assert(self.debugPrint("xnorTest(a=%s, b=%s)"%(a, b)))
    messenger.send(self.getName(), [(a and b) or (not (a or b))])

class LogicGateAI(Entity.Entity, PandaObject.PandaObject):
    if __debug__:
        notify = DirectNotifyGlobal.directNotify.newCategory(
                'LogicGateAI')
    logicTests={
        "and": andTest,
        "or": orTest,
        "xor": xorTest,
        "nand": nandTest,
        "nor": norTest,
        "xnor": xnorTest,
    }

    def __init__(self, level, entId):
        """entId: """
        assert(self.debugPrint(
                "LogicGateAI(entId=%s)"
                %(entId)))
        self.input1Event = None
        self.input2Event = None
        PandaObject.PandaObject.__init__(self)
        Entity.Entity.__init__(self, level, entId)
        self.setLogicType(self.logicType)
        self.setIsInput1(self.isInput1)
        self.setIsInput2(self.isInput2)
        self.setInput1Event(self.input1Event)
        self.setInput2Event(self.input2Event)

    def destroy(self):
        assert(self.debugPrint("destroy()"))
        self.ignore(self.input1Event)
        self.input1Event = None
        self.ignore(self.input2Event)
        self.input2Event = None
        Entity.Entity.destroy(self)

    def setLogicType(self, logicType):
        assert(self.debugPrint("setLogicType(logicType=%s)"%(logicType,)))
        self.logicType=logicType
        assert self.logicTests[logicType]
        self.logicTest=self.logicTests[logicType]

    def setIsInput1(self, isTrue):
        assert(self.debugPrint("setIsInput1(isTrue=%s)"%(isTrue,)))
        if 1 or (not isTrue) != (not self.input1Event):
            # ...the logical state of self.input1Event has changed.
            self.isInput1=isTrue
            self.logicTest(self, isTrue, self.isInput2)

    def setIsInput2(self, isTrue):
        assert(self.debugPrint("setIsInput1(isTrue=%s)"%(isTrue,)))
        if 1 or (not isTrue) != (not self.input2Event):
            # ...the logical state of self.input2Event has changed.
            self.isInput2=isTrue
            self.logicTest(self, isTrue, self.isInput1)

    def setInput1Event(self, event):
        assert(self.debugPrint("setInput1Event(event=%s)"%(event,)))
        if self.input1Event:
            self.ignore(self.input1Event)
        self.input1Event = "switch-%s"%(event,)
        if self.input1Event:
            self.accept(self.input1Event, self.setIsInput1)

    def setInput2Event(self, event):
        assert(self.debugPrint("setInput2Event(event=%s)"%(event,)))
        if self.input2Event:
            self.ignore(self.input2Event)
        self.input2Event = "switch-%s"%(event,)
        if self.input2Event:
            self.accept(self.input2Event, self.setIsInput2)

    def getName(self):
        #return "logicGate-%s"%(self.entId,)
        return "switch-%s"%(self.entId,)
<commit_msg>typo fix<commit_after>"""
LogicGateAI.py

    Logic Gates:

         and: 0 0 = 0     or: 0 0 = 0    xor: 0 0 = 0
              0 1 = 0         0 1 = 1         0 1 = 1
              1 0 = 0         1 0 = 1         1 0 = 1
              1 1 = 1         1 1 = 1         1 1 = 0

        nand: 0 0 = 1    nor: 0 0 = 1   xnor: 0 0 = 1
              0 1 = 1         0 1 = 0         0 1 = 0
              1 0 = 1         1 0 = 0         1 0 = 0
              1 1 = 0         1 1 = 0         1 1 = 1

    In the following:
        1: send a true message
        0: send a false message
        -: don't send a message

        a b  and  or  xor  nand  nor  xnor
       (0 0)  (0) (0)  (0)   (1)  (1)   (1)  <--- initial state
        1 0    -   1    1     -    0     0
        0 0    -   0    0     -    1     1
        1 0    -   1    1     -    0     0
        1 1    1   -    0     0    -     1
        0 1    0   -    1     1    -     0
        1 1    1   -    0     0    -     1
        0 1    0   -    1     1    -     0
        0 0    -   0    0     -    1     1
"""

import PandaObject
import DirectNotifyGlobal
import Entity

def andTest(self, a, b):
    assert(self.debugPrint("andTest(a=%s, b=%s)"%(a, b)))
    if b:
        messenger.send(self.getName(), [a])

def orTest(self, a, b):
    assert(self.debugPrint("orTest(a=%s, b=%s)"%(a, b)))
    if not b:
        messenger.send(self.getName(), [a])
    # else: ...we already sent the messege when b was set.

def xorTest(self, a, b):
    assert(self.debugPrint("xorTest(a=%s, b=%s)"%(a, b)))
    messenger.send(self.getName(), [(not (a and b)) and (a or b)])

def nandTest(self, a, b):
    assert(self.debugPrint("nandTest(a=%s, b=%s)"%(a, b)))
    if b:
        messenger.send(self.getName(), [not (a and b)])

def norTest(self, a, b):
    assert(self.debugPrint("norTest(a=%s, b=%s)"%(a, b)))
    if not b:
        messenger.send(self.getName(), [not (a or b)])
    # else: ...we already sent the messege when b was set.

def xnorTest(self, a, b):
    assert(self.debugPrint("xnorTest(a=%s, b=%s)"%(a, b)))
    messenger.send(self.getName(), [(a and b) or (not (a or b))])

class LogicGateAI(Entity.Entity, PandaObject.PandaObject):
    if __debug__:
        notify = DirectNotifyGlobal.directNotify.newCategory(
                'LogicGateAI')
    logicTests={
        "and": andTest,
        "or": orTest,
        "xor": xorTest,
        "nand": nandTest,
        "nor": norTest,
        "xnor": xnorTest,
    }

    def __init__(self, level, entId):
        """entId: """
        assert(self.debugPrint(
                "LogicGateAI(entId=%s)"
                %(entId)))
        self.input1Event = None
        self.input2Event = None
        PandaObject.PandaObject.__init__(self)
        Entity.Entity.__init__(self, level, entId)
        self.setLogicType(self.logicType)
        self.setIsInput1(self.isInput1)
        self.setIsInput2(self.isInput2)
        self.setInput1Event(self.input1Event)
        self.setInput2Event(self.input2Event)

    def destroy(self):
        assert(self.debugPrint("destroy()"))
        self.ignore(self.input1Event)
        self.input1Event = None
        self.ignore(self.input2Event)
        self.input2Event = None
        Entity.Entity.destroy(self)

    def setLogicType(self, logicType):
        assert(self.debugPrint("setLogicType(logicType=%s)"%(logicType,)))
        self.logicType=logicType
        assert self.logicTests[logicType]
        self.logicTest=self.logicTests[logicType]

    def setIsInput1(self, isTrue):
        assert(self.debugPrint("setIsInput1(isTrue=%s)"%(isTrue,)))
        if 1 or (not isTrue) != (not self.input1Event):
            # ...the logical state of self.input1Event has changed.
            self.isInput1=isTrue
            self.logicTest(self, isTrue, self.isInput2)

    def setIsInput2(self, isTrue):
        assert(self.debugPrint("setIsInput2(isTrue=%s)"%(isTrue,)))
        if 1 or (not isTrue) != (not self.input2Event):
            # ...the logical state of self.input2Event has changed.
            self.isInput2=isTrue
            self.logicTest(self, isTrue, self.isInput1)

    def setInput1Event(self, event):
        assert(self.debugPrint("setInput1Event(event=%s)"%(event,)))
        if self.input1Event:
            self.ignore(self.input1Event)
        self.input1Event = "switch-%s"%(event,)
        if self.input1Event:
            self.accept(self.input1Event, self.setIsInput1)

    def setInput2Event(self, event):
        assert(self.debugPrint("setInput2Event(event=%s)"%(event,)))
        if self.input2Event:
            self.ignore(self.input2Event)
        self.input2Event = "switch-%s"%(event,)
        if self.input2Event:
            self.accept(self.input2Event, self.setIsInput2)

    def getName(self):
        #return "logicGate-%s"%(self.entId,)
        return "switch-%s"%(self.entId,)
<|endoftext|>


<commit_before>import os
import pytest
from great_expectations.render.renderer.site_builder import SiteBuilder

def test_cli_profile(titanic_data_context):
    print("BOOO")

    titanic_data_context.profile_datasource(titanic_data_context.list_datasources()[0]["name"])

    site_config = {
        "site_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/documentation/local_site"
        },
        "validations_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/validations/",
            "run_id_filter": {
                "ne": "profiling"
            }
        },
        "profiling_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/validations/",
            "run_id_filter": {
                "eq": "profiling"
            }
        },
        "datasources": "*",
        "sections": {
            "index": {
                "renderer": {
                    "module": "great_expectations.render.renderer",
                    "class": "SiteIndexPageRenderer"
                }
            },
            "expectations": {
                "renderer": {
                    "module": "great_expectations.render.renderer",
                    "class": "ExpectationSuitePageRenderer"
                },
                "view": {
                    "module": "great_expectations.render.view",
                    "class": "DefaultJinjaPageView"
                }
            },
        }

    }

    res = SiteBuilder.build(titanic_data_context, site_config)
    index_page_locator_info = res[0]
    index_links_dict = res[1]

    assert index_page_locator_info['path'] == titanic_data_context.data_doc_directory + '/local_site/index.html'
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['expectation_suite_links']) == 1
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['validation_links']) == 0
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['profiling_links']) == 0

<commit_msg>Removed a forgotten BOOO print :)<commit_after>import os
import pytest
from great_expectations.render.renderer.site_builder import SiteBuilder

def test_cli_profile(titanic_data_context):

    titanic_data_context.profile_datasource(titanic_data_context.list_datasources()[0]["name"])

    site_config = {
        "site_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/documentation/local_site"
        },
        "validations_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/validations/",
            "run_id_filter": {
                "ne": "profiling"
            }
        },
        "profiling_store": {
            "type": "filesystem",
            "base_directory": "uncommitted/validations/",
            "run_id_filter": {
                "eq": "profiling"
            }
        },
        "datasources": "*",
        "sections": {
            "index": {
                "renderer": {
                    "module": "great_expectations.render.renderer",
                    "class": "SiteIndexPageRenderer"
                }
            },
            "expectations": {
                "renderer": {
                    "module": "great_expectations.render.renderer",
                    "class": "ExpectationSuitePageRenderer"
                },
                "view": {
                    "module": "great_expectations.render.view",
                    "class": "DefaultJinjaPageView"
                }
            },
        }

    }

    res = SiteBuilder.build(titanic_data_context, site_config)
    index_page_locator_info = res[0]
    index_links_dict = res[1]

    assert index_page_locator_info['path'] == titanic_data_context.data_doc_directory + '/local_site/index.html'
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['expectation_suite_links']) == 1
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['validation_links']) == 0
    assert len(index_links_dict['mydatasource']['mygenerator']['Titanic']['profiling_links']) == 0

<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import eventlet
import requests

from flask import jsonify, request, Flask
from six.moves import http_client
from six.moves import urllib_parse
from st2reactor.sensor.base import Sensor

PACK = 'newrelic'
WEB_APP_ALERT_TRIGGER_REF = '{}.{}'.format(PACK, 'WebAppAlertTrigger')
WEB_APP_NORMAL_TRIGGER_REF = '{}.{}'.format(PACK, 'WebAppNormalTrigger')
SERVER_ALERT_TRIGGER_REF = '{}.{}'.format(PACK, 'ServerAlertTrigger')
SERVER_NORMAL_TRIGGER_REF = '{}.{}'.format(PACK, 'ServerNormalTrigger')

NR_API_URL_KEY = 'api_url'
NR_API_KEY_KEY = 'api_key'

APP_HOST_KEY = 'host'
APP_PORT_KEY = 'port'
APP_URL_KEY = 'url'
NORMAL_REPORT_DELAY_KEY = 'normal_report_delay'

class NewRelicHookSensor(Sensor):

    def __init__(self, sensor_service, config=None):
        self._config = config
        self._sensor_service = sensor_service

        self._api_url = config.get(NR_API_URL_KEY, None)
        self._api_key = config.get(NR_API_KEY_KEY, None)

        self._host = self._get_sensor_config_param(self._config, APP_HOST_KEY)
        self._port = self._get_sensor_config_param(self._config, APP_PORT_KEY)
        self._url = self._get_sensor_config_param(self._config, APP_URL_KEY)
        self._normal_report_delay = self._get_sensor_config_param(self._config, 300)

        self._app = Flask(__name__)
        self._log = self._sensor_service.get_logger(__name__)
        self._headers = {'X-Api-Key': self._api_key}

    def setup(self):
        pass

    def run(self):
        if not self._api_key:
            raise Exception('NewRelic API key not found.')
        if not self._host or not self._port or not self._url:
            raise Exception('NewRelic webhook app config (host:%s, port:%s, url:%s)' %
                            (self._host, self._port, self._url))
        self._log.info('NewRelicHookSensor up. host %s, port %s, url %s', self._host, self._port,
                       self._url)
        self._app.add_url_rule(self._url, None, self._handle_nrhook, methods=['POST'])
        self._app.run(host=self._host, port=self._port)

    def _handle_nrhook(self, url):

        # hooks are sent for alerts and deployments. Only care about alerts so ignoring
        # deployments. Body expected to be of the form -
        #
        # alert : {...}
        #      OR
        # deployment : {...}
        #
        alert_body = request.get_json().get('alert', None)

        hook_headers = self._get_headers_as_dict(request.headers)
        hook_handler = self._get_hook_handler(alert_body, hook_headers)

        # all handling based off 'docs' found in this documentation -
        # https://docs.newrelic.com/docs/alerts/alert-policies/examples/webhook-examples

        try:
            if hook_handler:
                hook_handler(alert_body, hook_headers)
        except Exception:
            self._log.Exception('Failed to handle nr hook %s.', alert_body)

        # payload = {}
        # payload['headers'] = hook_headers
        # payload['body'] = hook_body
        # self._dispatch_payload(payload)

        return jsonify({}, http_client.ACCEPTED)

    def _get_hook_handler(self, alert_body, hook_headers):
        if not alert_body:
            return None

        # (todo) include severity check

        if 'servers' in alert_body:
            return self._server_hook_handler
        # For now everything else is web app hook. Hooks for key transaction, mobile app or plugin
        # alert all would be rolled up the application level.
        return self._app_hook_handler

    def _app_hook_handler(self, alert_body, hook_headers):

        long_description = alert_body['long_description']

        if self._is_alert_opened(long_description) or \
           self._is_escalated_downtime(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            self._sensor_service.dispatch(WEB_APP_ALERT_TRIGGER_REF, payload)

        elif self._is_alert_closed(long_description) or \
             self._is_downtime_recovered(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_application_normal,
                                 payload)

        elif self._is_alert_canceled(long_description) or \
             self._is_alert_acknowledged(long_description):
            self._log.info('Ignored alert : %s.', alert_body)

    def _dispatch_application_normal(self, payload):
        '''
        '''
        application = self._get_application(payload['alert']['application_name'])
        if application['health_status'] in ['green']:
            self._sensor_service.dispatch(WEB_APP_NORMAL_TRIGGER_REF, payload)
        else:
            self._log.info('Application %s has state %s. Rescheduling normal check.',
                           application['name'], application['health_status'])
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_application_normal,
                                 payload)

    def _server_hook_handler(self, alert_body, hook_headers):
        long_description = alert_body['long_description']
        if self._is_alert_opened(long_description) or \
           self._is_escalated_downtime(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            self._sensor_service.dispatch(SERVER_ALERT_TRIGGER_REF, payload)

        elif self._is_alert_closed(long_description) or \
             self._is_downtime_recovered(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_server_normal,
                                 payload)

        elif self._is_alert_canceled(long_description) or \
             self._is_alert_acknowledged(long_description):
            self._log.info('Ignored alert : %s.', alert_body)

    def _dispatch_server_normal(self, payload):
        '''
        '''
        servers = self._get_servers(payload['alert']['servers'])
        # make sure all servers are ok.
        all_servers_ok = True
        for server in servers:
            all_servers_ok &= server['health_status'] in ['green']
            if not all_servers_ok:
                break

        if all_servers_ok:
            self._sensor_service.dispatch(WEB_APP_NORMAL_TRIGGER_REF, payload)
        else:
            for server in servers:
                self._log.info('server %s has state %s. Rescheduling normal check.',
                               server['name'], server['health_status'])
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_server_normal,
                                 payload)

    # alert test methods
    def _is_alert_opened(self, long_description):
        return long_description and long_description.startswith('Alert opened')

    def _is_alert_closed(self, long_description):
        return long_description and long_description.startswith('Alert ended')

    def _is_alert_canceled(self, long_description):
        return long_description and long_description.startswith('Alert canceled')

    def _is_alert_acknowledged(self, long_description):
        return long_description and long_description.startswith('Alert acknowledged')

    def _is_escalated_downtime(self, long_description):
        return long_description and long_description.startswith('Alert escalated to downtime')

    def _is_downtime_recovered(self, long_description):
        return long_description and long_description.startswith('Alert downtime recovered')

    # newrelic API methods
    def _get_application(self, app_name):
        params = None
        if app_name:
            params = {'filter[name]': app_name}
        url = urllib_parse.urljoin(self._api_url, 'applications.json')
        resp = requests.get(url, headers=self._headers, params=params).json()
        if 'applications' in resp:
            # pick 1st application
            return resp['applications'][0] if resp['applications'] else None
        return None

    def _get_servers(self, server_names):
        servers = {}
        # No batch query by name support so making API calls in a tight loop. Might be
        # ok to get all severs and filter manually but that gets complex for a large number
        # of server since the API pages data.
        for server_name in server_names:
            params = {'filter[name]': server_name}
            url = urllib_parse.urljoin(self._api_url, 'servers.json')
            resp = requests.get(url, headers=self._headers, params=params).json()
            servers['server_name'] = resp['servers'][0] if resp['servers'] else None
        return servers

    def cleanup(self):
        # self._log.info('cleanup called.')
        # # If Flask is using the default Werkzeug server, then call shutdown on it.
        # func = request.environ.get('werkzeug.server.shutdown')
        # if func is None:
        #     raise Exception('Not running with the Werkzeug Server')
        # func()
        pass

    @staticmethod
    def _get_sensor_config_param(config, param_name, default=None):
        sensor_config = NewRelicHookSensor._get_sensor_config(config)
        if sensor_config:
            return sensor_config.get(param_name, default)
        return default

    @staticmethod
    def _get_sensor_config(config):
        return config.get('sensor_config', None)

    @staticmethod
    def _get_headers_as_dict(headers):
        headers_dict = {}
        for k, v in headers:
            headers_dict[k] = v
        return headers_dict

    # ignore
    def add_trigger(self, trigger):
        pass

    def update_trigger(self, trigger):
        pass

    def remove_trigger(self, trigger):
        pass
<commit_msg>Docs and comments.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import eventlet
import requests

from flask import jsonify, request, Flask
from six.moves import http_client
from six.moves import urllib_parse
from st2reactor.sensor.base import Sensor

PACK = 'newrelic'
WEB_APP_ALERT_TRIGGER_REF = '{}.{}'.format(PACK, 'WebAppAlertTrigger')
WEB_APP_NORMAL_TRIGGER_REF = '{}.{}'.format(PACK, 'WebAppNormalTrigger')
SERVER_ALERT_TRIGGER_REF = '{}.{}'.format(PACK, 'ServerAlertTrigger')
SERVER_NORMAL_TRIGGER_REF = '{}.{}'.format(PACK, 'ServerNormalTrigger')

NR_API_URL_KEY = 'api_url'
NR_API_KEY_KEY = 'api_key'

APP_HOST_KEY = 'host'
APP_PORT_KEY = 'port'
APP_URL_KEY = 'url'
NORMAL_REPORT_DELAY_KEY = 'normal_report_delay'

class NewRelicHookSensor(Sensor):
    """
    Sensor class that starts up a flask webapp that listens to alert hooks from NewRelic.
    It translates hooks into appropriate triggers using the following mapping -
       1. Web app incident and apdex problem opened -> WEB_APP_ALERT_TRIGGER
       2. Incident escalated to downtime (app)      -> WEB_APP_ALERT_TRIGGER
       3. Apdex problem closed (app)                -> WEB_APP_NORMAL_TRIGGER_REF
       4. Downtime problem closed (app)             -> WEB_APP_NORMAL_TRIGGER_REF
       5. Server incident and CPU problem opened    -> SERVER_ALERT_TRIGGER_REF
       6. Incident escalated after 5 minutes        -> SERVER_ALERT_TRIGGER_REF
       7. Server downtime ends                      -> SERVER_NORMAL_TRIGGER_REF
       8. CPU problem closed                        -> SERVER_NORMAL_TRIGGER_REF

    Note : Some hooks like cancel or disable of an inciden and open or close of alert policy
    are ignored.

    All return to normal events are always fired after a delay period.
    """

    def __init__(self, sensor_service, config=None):
        self._config = config
        self._sensor_service = sensor_service

        self._api_url = config.get(NR_API_URL_KEY, None)
        self._api_key = config.get(NR_API_KEY_KEY, None)

        self._host = self._get_sensor_config_param(self._config, APP_HOST_KEY)
        self._port = self._get_sensor_config_param(self._config, APP_PORT_KEY)
        self._url = self._get_sensor_config_param(self._config, APP_URL_KEY)
        self._normal_report_delay = self._get_sensor_config_param(self._config, 300)

        self._app = Flask(__name__)
        self._log = self._sensor_service.get_logger(__name__)
        self._headers = {'X-Api-Key': self._api_key}

    def setup(self):
        pass

    def run(self):
        """
        Validate required params and starts up the webapp that listen to hooks from NewRelic.
        """
        if not self._api_url:
            raise Exception('NewRelic API url not found.')
        if not self._api_key:
            raise Exception('NewRelic API key not found.')
        if not self._host or not self._port or not self._url:
            raise Exception('NewRelic webhook app config (host:%s, port:%s, url:%s)' %
                            (self._host, self._port, self._url))
        self._log.info('NewRelicHookSensor up. host %s, port %s, url %s', self._host, self._port,
                       self._url)
        self._app.add_url_rule(self._url, None, self._handle_nrhook, methods=['POST'])
        self._app.run(host=self._host, port=self._port)

    def _handle_nrhook(self, url):

        # hooks are sent for alerts and deployments. Only care about alerts so ignoring
        # deployments. Body expected to be of the form -
        #
        # alert : {...}
        #      OR
        # deployment : {...}
        #
        alert_body = request.get_json().get('alert', None)

        if alert_body.get('severity', None) not in ['critical', 'downtime']:
            self._log.debug('Ignoring alert %s as it is not severe enough.', alert_body)
            return jsonify({}, http_client.ACCEPTED)

        hook_headers = self._get_headers_as_dict(request.headers)
        hook_handler = self._get_hook_handler(alert_body, hook_headers)

        # all handling based off 'docs' found in this documentation -
        # https://docs.newrelic.com/docs/alerts/alert-policies/examples/webhook-examples

        try:
            if hook_handler:
                hook_handler(alert_body, hook_headers)
        except Exception:
            self._log.Exception('Failed to handle nr hook %s.', alert_body)

        return jsonify({}, http_client.ACCEPTED)

    def _get_hook_handler(self, alert_body, hook_headers):
        if not alert_body:
            return None

        if 'servers' in alert_body:
            return self._server_hook_handler

        # For now everything else is web app hook. Hooks for key transaction, mobile app or plugin
        # alert all would be rolled up the application level.
        return self._app_hook_handler

    def _app_hook_handler(self, alert_body, hook_headers):

        if not alert_body['application_name']:
            self._log.info('No application found for alert %s. Will Ignore.', alert_body)
            return

        long_description = alert_body['long_description']

        if self._is_alert_opened(long_description) or \
           self._is_escalated_downtime(long_description):

            # handled opened and escalation to downtime immediately.
            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            self._sensor_service.dispatch(WEB_APP_ALERT_TRIGGER_REF, payload)

        elif self._is_alert_closed(long_description) or \
             self._is_downtime_recovered(long_description):

            # handled closed and recovered after a delay.
            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_application_normal,
                                 payload)

        elif self._is_alert_canceled(long_description) or \
             self._is_alert_acknowledged(long_description):

            # ignore canceled or acknowledged
            self._log.info('Ignored alert : %s.', alert_body)

    def _dispatch_application_normal(self, payload, attempt_no=0):
        '''
        Dispatches WEB_APP_NORMAL_TRIGGER_REF if the application health_status is 'green'.
        '''
        # basic guard to avoid queuing up forever.
        if attempt_no == 10:
            self._log.warn('Abandoning WEB_APP_NORMAL_TRIGGER_REF dispatch. Payload %s', payload)
        application = self._get_application(payload['alert']['application_name'])
        if application['health_status'] in ['green']:
            self._sensor_service.dispatch(WEB_APP_NORMAL_TRIGGER_REF, payload)
        else:
            self._log.info('Application %s has state %s. Rescheduling normal check.',
                           application['name'], application['health_status'])
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_application_normal,
                                 payload, attempt_no + 1)

    def _server_hook_handler(self, alert_body, hook_headers):
        long_description = alert_body['long_description']
        if self._is_alert_opened(long_description) or \
           self._is_escalated_downtime(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            self._sensor_service.dispatch(SERVER_ALERT_TRIGGER_REF, payload)

        elif self._is_alert_closed(long_description) or \
             self._is_downtime_recovered(long_description):

            payload = {
                'alert': alert_body,
                'header': hook_headers
            }
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_server_normal,
                                 payload)

        elif self._is_alert_canceled(long_description) or \
             self._is_alert_acknowledged(long_description):
            self._log.info('Ignored alert : %s.', alert_body)

    def _dispatch_server_normal(self, payload, attempt_no=0):
        '''
        Dispatches SERVER_NORMAL_TRIGGER_REF if the all servers health_status is 'green'.
        '''
        # basic guard to avoid queuing up forever.
        if attempt_no == 10:
            self._log.warn('Abandoning SERVER_NORMAL_TRIGGER_REF dispatch. Payload %s', payload)
        servers = self._get_servers(payload['alert']['servers'])
        # make sure all servers are ok.
        all_servers_ok = True
        for server in servers:
            all_servers_ok &= server['health_status'] in ['green']
            if not all_servers_ok:
                break

        if all_servers_ok:
            self._sensor_service.dispatch(WEB_APP_NORMAL_TRIGGER_REF, payload)
        else:
            for server in servers:
                self._log.info('server %s has state %s. Rescheduling normal check.',
                               server['name'], server['health_status'])
            eventlet.spawn_after(self._normal_report_delay, self._dispatch_server_normal,
                                 payload, attempt_no + 1)

    # alert test methods
    def _is_alert_opened(self, long_description):
        return long_description and long_description.startswith('Alert opened')

    def _is_alert_closed(self, long_description):
        return long_description and long_description.startswith('Alert ended')

    def _is_alert_canceled(self, long_description):
        return long_description and long_description.startswith('Alert canceled')

    def _is_alert_acknowledged(self, long_description):
        return long_description and long_description.startswith('Alert acknowledged')

    def _is_escalated_downtime(self, long_description):
        return long_description and long_description.startswith('Alert escalated to downtime')

    def _is_downtime_recovered(self, long_description):
        return long_description and long_description.startswith('Alert downtime recovered')

    # newrelic API methods
    def _get_application(self, app_name):
        params = None
        if app_name:
            params = {'filter[name]': app_name}
        url = urllib_parse.urljoin(self._api_url, 'applications.json')
        resp = requests.get(url, headers=self._headers, params=params).json()
        if 'applications' in resp:
            # pick 1st application
            return resp['applications'][0] if resp['applications'] else None
        return None

    def _get_servers(self, server_names):
        servers = {}
        # No batch query by name support so making API calls in a tight loop. Might be
        # ok to get all severs and filter manually but that gets complex for a large number
        # of server since the API pages data.
        for server_name in server_names:
            params = {'filter[name]': server_name}
            url = urllib_parse.urljoin(self._api_url, 'servers.json')
            resp = requests.get(url, headers=self._headers, params=params).json()
            servers['server_name'] = resp['servers'][0] if resp['servers'] else None
        return servers

    def cleanup(self):
        # self._log.info('cleanup called.')
        # # If Flask is using the default Werkzeug server, then call shutdown on it.
        # func = request.environ.get('werkzeug.server.shutdown')
        # if func is None:
        #     raise Exception('Not running with the Werkzeug Server')
        # func()
        pass

    @staticmethod
    def _get_sensor_config_param(config, param_name, default=None):
        sensor_config = NewRelicHookSensor._get_sensor_config(config)
        if sensor_config:
            return sensor_config.get(param_name, default)
        return default

    @staticmethod
    def _get_sensor_config(config):
        return config.get('sensor_config', None)

    @staticmethod
    def _get_headers_as_dict(headers):
        headers_dict = {}
        for k, v in headers:
            headers_dict[k] = v
        return headers_dict

    # ignore
    def add_trigger(self, trigger):
        pass

    def update_trigger(self, trigger):
        pass

    def remove_trigger(self, trigger):
        pass
<|endoftext|>


<commit_before>import pandas

class ApplianceTrace(object):

    def __init__(self, series, source):
        '''
        Initializes a trace object from a series and a source.
        Series must be sampled at a particular sample rate
        '''
        self.series = series
        self.source = source

    def get_series(self):
        return self.series

    def get_source(self):
        return self.series

    def get_sampling_rate(self):
        return self.series.index.freq
<commit_msg>update appliance trace<commit_after>import pandas

class ApplianceTrace(object):

    def __init__(self, series, source):
        '''
        Initializes a trace object from a series and a source.
        Series must be sampled at a particular sample rate
        '''
        self.series = series
        self.source = source

    def get_sampling_rate(self):
        '''
        Returns a string representing the rate at which the series was sampled.
        '''
        return self.series.index.freq

    def get_series(self):
        '''Returns the pandas series object representing this trace.'''
        return self.series

    def get_source(self):
        '''Returns the user-supplied trace source string.'''
        return self.source

    def set_series(self,series):
        '''Updates the series (such as after a resampling)'''
        self.series = series

    def set_source(self,source):
        '''Updates the user-supplied source string.'''
        self.source = source
<|endoftext|>