<commit_before># Copyright 2020 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Computes log-ratios of Jacobian determinants numerically stably."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import inspect

import tensorflow.compat.v2 as tf

from tensorflow_probability.python.bijectors import bijector as bijector_lib
from tensorflow_probability.python.internal import prefer_static as ps

__all__ = [
    'forward_log_det_jacobian_ratio',
    'inverse_log_det_jacobian_ratio',
    'RegisterFLDJRatio',
    'RegisterILDJRatio',
]

_fldj_ratio_registry = {}
_ildj_ratio_registry = {}

def _reduce_ldj_ratio(unreduced_ldj_ratio, p, q, input_shape, min_event_ndims,
                      event_ndims):
  """Reduces an LDJ ratio computed with event_ndims=min_event_ndims."""
  # pylint: disable=protected-access
  have_parameter_batch_shape = (
      p._parameter_batch_shape is not None and
      q._parameter_batch_shape is not None)
  if have_parameter_batch_shape:
    parameter_batch_shape = ps.broadcast_shape(
        p._parameter_batch_shape,
        q._parameter_batch_shape)
  else:
    parameter_batch_shape = None

  reduce_shape, assertions = bijector_lib.ldj_reduction_shape(
      input_shape,
      event_ndims=event_ndims,
      min_event_ndims=min_event_ndims,
      parameter_batch_shape=parameter_batch_shape,
      allow_event_shape_broadcasting=not (p._parts_interact or
                                          q._parts_interact),
      validate_args=p.validate_args or q.validate_args)

  sum_fn = getattr(p, '_sum_fn', getattr(q, '_sum_fn', tf.reduce_sum))
  with tf.control_dependencies(assertions):
    return bijector_lib.reduce_jacobian_det_over_shape(
        unreduced_ldj_ratio, reduce_shape=reduce_shape, sum_fn=sum_fn)

def _default_fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
  min_event_ndims = p.forward_min_event_ndims
  unreduced_fldj_ratio = (
      p.forward_log_det_jacobian(x, event_ndims=min_event_ndims, **p_kwargs) -
      q.forward_log_det_jacobian(y, event_ndims=min_event_ndims, **q_kwargs))
  return _reduce_ldj_ratio(unreduced_fldj_ratio, p, q, ps.shape(x),
                           min_event_ndims, event_ndims)

def _default_ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
  min_event_ndims = p.inverse_min_event_ndims
  unreduced_fldj_ratio = (
      p.inverse_log_det_jacobian(x, event_ndims=event_ndims, **p_kwargs) -
      q.inverse_log_det_jacobian(y, event_ndims=event_ndims, **q_kwargs))
  return _reduce_ldj_ratio(unreduced_fldj_ratio, p, q, ps.shape(x),
                           min_event_ndims, event_ndims)

def _get_ldj_ratio_fn(cls, registry):
  ldj_ratio_fn = None
  for ref_cls in inspect.getmro(cls):
    if ref_cls in registry:
      ldj_ratio_fn = registry[ref_cls]
      break
  return ldj_ratio_fn

def forward_log_det_jacobian_ratio(
    p,
    x,
    q,
    y,
    event_ndims,
    p_kwargs=None,
    q_kwargs=None,
):
  """Computes `p.fldj(x, ndims) - q.fdlj(y, ndims)`, numerically stably.

  `p_kwargs` and `q_kwargs` are passed to the registered `fldj_ratio_fn`. The
  fallback implementation passes them to the `forward_log_det_jacobian` methods
  of `p` and `q`.

  Args:
    p: A bijector instance.
    x: A tensor from the preimage of `p.forward`.
    q: A bijector instance of the same type as `p`, with matching shape.
    y: A tensor from the preimage of `q.forward`.
    event_ndims: The number of right-hand dimensions comprising the event shapes
      of `x` and `y`.
    p_kwargs: Keyword args to pass to `p`.
    q_kwargs: Keyword args to pass to `q`.

  Returns:
    fldj_ratio: `log ((abs o det o jac p)(x) / (abs o det o jac q)(y))`,
      i.e. in TFP code, `p.forward_log_det_jacobian(x, event_ndims) -
      q.forward_log_det_jacobian(y, event_ndims)`. In some cases
      this will be computed with better than naive numerical precision, e.g. by
      moving differences inside of a sum reduction.
  """
  assert type(p) == type(q)  # pylint: disable=unidiomatic-typecheck
  if p_kwargs is None:
    p_kwargs = {}
  if q_kwargs is None:
    q_kwargs = {}

  fldj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_fldj_ratio_registry)
  ildj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_ildj_ratio_registry)

  def inverse_fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
    # p.fldj(x) - q.fldj(y) = q.ildj(q(y)) - p.ildj(p(x))
    return ildj_ratio_fn(q, q.forward(y), p, p.forward(x),
                         q.forward_event_ndims(event_ndims), q_kwargs, p_kwargs)

  if fldj_ratio_fn is None:
    if ildj_ratio_fn is None:
      fldj_ratio_fn = _default_fldj_ratio_fn
    else:
      fldj_ratio_fn = inverse_fldj_ratio_fn

  return fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs)

def inverse_log_det_jacobian_ratio(
    p,
    x,
    q,
    y,
    event_ndims,
    p_kwargs=None,
    q_kwargs=None,
):
  """Computes `p.ildj(x, ndims) - q.idlj(y, ndims)`, numerically stably.

  `p_kwargs` and `q_kwargs` are passed to the registered `ildj_ratio_fn`. The
  fallback implementation passes them to the `inverse_log_det_jacobian` methods
  of `p` and `q`.

  Args:
    p: A bijector instance.
    x: A tensor from the image of `p.forward`.
    q: A bijector instance of the same type as `p`, with matching shape.
    y: A tensor from the image of `q.forward`.
    event_ndims: The number of right-hand dimensions comprising the event shapes
      of `x` and `y`.
    p_kwargs: Keyword args to pass to `p`.
    q_kwargs: Keyword args to pass to `q`.

  Returns:
    ildj_ratio: `log ((abs o det o jac p^-1)(x) / (abs o det o jac q^-1)(y))`,
      i.e. in TFP code, `p.inverse_log_det_jacobian(x, event_ndims) -
      q.inverse_log_det_jacobian(y, event_ndims)`. In some cases
      this will be computed with better than naive numerical precision, e.g. by
      moving differences inside of a sum reduction.
  """
  assert type(p) == type(q)  # pylint: disable=unidiomatic-typecheck
  if p_kwargs is None:
    p_kwargs = {}
  if q_kwargs is None:
    q_kwargs = {}

  ildj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_ildj_ratio_registry)
  fldj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_fldj_ratio_registry)

  def inverse_ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
    # p.ildj(x) - q.ildj(y) = q.fldj(q^-1(y)) - p.fldj(p^-1(x))
    return fldj_ratio_fn(q, q.inverse(y), p, p.inverse(x),
                         q.inverse_event_ndims(event_ndims), q_kwargs, p_kwargs)

  if ildj_ratio_fn is None:
    if fldj_ratio_fn is None:
      ildj_ratio_fn = _default_ildj_ratio_fn
    else:
      ildj_ratio_fn = inverse_ildj_ratio_fn

  if tf.nest.is_nested(p.inverse_min_event_ndims):
    # See the comment in forward_log_det_jacobian_ratio about why we do this.
    return ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs)
  else:
    # Evaluate the ratio at minimum event ndims, and then reduce the unreduced
    # LDJ.
    min_event_ndims = p.inverse_min_event_ndims
    have_parameter_batch_shape = (
        p._parameter_batch_shape is not None and  # pylint: disable=protected-access
        q._parameter_batch_shape is not None)  # pylint: disable=protected-access
    reduce_shape, assertions = bijector_lib.ldj_reduction_shape(
        ps.shape(x),
        event_ndims=event_ndims,
        min_event_ndims=min_event_ndims,
        parameter_batch_shape=(ps.broadcast_shape(p._parameter_batch_shape,  # pylint: disable=protected-access
                                                  q._parameter_batch_shape)  # pylint: disable=protected-access
                               if have_parameter_batch_shape else None),
        allow_event_shape_broadcasting=True,
        validate_args=p.validate_args or q.validate_args)

    sum_fn = getattr(p, '_sum_fn', getattr(q, '_sum_fn', tf.reduce_sum))
    with tf.control_dependencies(assertions):
      return bijector_lib.reduce_jacobian_det_over_shape(
          ildj_ratio_fn(p, x, q, y, min_event_ndims, p_kwargs, q_kwargs),
          reduce_shape=reduce_shape,
          sum_fn=sum_fn)

class RegisterFLDJRatio(object):

  def __init__(self, bijector_class):
    self.cls = bijector_class

  def __call__(self, fn):
    assert self.cls not in _fldj_ratio_registry
    _fldj_ratio_registry[self.cls] = fn
    return fn

class RegisterILDJRatio(object):

  def __init__(self, bijector_class):
    self.cls = bijector_class

  def __call__(self, fn):
    assert self.cls not in _ildj_ratio_registry
    _ildj_ratio_registry[self.cls] = fn
    return fn
<commit_msg>Remove unnecessary code from ldj_ratio.py<commit_after># Copyright 2020 The TensorFlow Probability Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Computes log-ratios of Jacobian determinants numerically stably."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import inspect

import tensorflow.compat.v2 as tf

from tensorflow_probability.python.bijectors import bijector as bijector_lib
from tensorflow_probability.python.internal import prefer_static as ps

__all__ = [
    'forward_log_det_jacobian_ratio',
    'inverse_log_det_jacobian_ratio',
    'RegisterFLDJRatio',
    'RegisterILDJRatio',
]

_fldj_ratio_registry = {}
_ildj_ratio_registry = {}

def _reduce_ldj_ratio(unreduced_ldj_ratio, p, q, input_shape, min_event_ndims,
                      event_ndims):
  """Reduces an LDJ ratio computed with event_ndims=min_event_ndims."""
  # pylint: disable=protected-access
  have_parameter_batch_shape = (
      p._parameter_batch_shape is not None and
      q._parameter_batch_shape is not None)
  if have_parameter_batch_shape:
    parameter_batch_shape = ps.broadcast_shape(p._parameter_batch_shape,
                                               q._parameter_batch_shape)
  else:
    parameter_batch_shape = None

  reduce_shape, assertions = bijector_lib.ldj_reduction_shape(
      input_shape,
      event_ndims=event_ndims,
      min_event_ndims=min_event_ndims,
      parameter_batch_shape=parameter_batch_shape,
      allow_event_shape_broadcasting=not (p._parts_interact or
                                          q._parts_interact),
      validate_args=p.validate_args or q.validate_args)

  sum_fn = getattr(p, '_sum_fn', getattr(q, '_sum_fn', tf.reduce_sum))
  with tf.control_dependencies(assertions):
    return bijector_lib.reduce_jacobian_det_over_shape(
        unreduced_ldj_ratio, reduce_shape=reduce_shape, sum_fn=sum_fn)

def _default_fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
  min_event_ndims = p.forward_min_event_ndims
  unreduced_fldj_ratio = (
      p.forward_log_det_jacobian(x, event_ndims=min_event_ndims, **p_kwargs) -
      q.forward_log_det_jacobian(y, event_ndims=min_event_ndims, **q_kwargs))
  return _reduce_ldj_ratio(unreduced_fldj_ratio, p, q, ps.shape(x),
                           min_event_ndims, event_ndims)

def _default_ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
  min_event_ndims = p.inverse_min_event_ndims
  unreduced_fldj_ratio = (
      p.inverse_log_det_jacobian(x, event_ndims=min_event_ndims, **p_kwargs) -
      q.inverse_log_det_jacobian(y, event_ndims=min_event_ndims, **q_kwargs))
  return _reduce_ldj_ratio(unreduced_fldj_ratio, p, q, ps.shape(x),
                           min_event_ndims, event_ndims)

def _get_ldj_ratio_fn(cls, registry):
  ldj_ratio_fn = None
  for ref_cls in inspect.getmro(cls):
    if ref_cls in registry:
      ldj_ratio_fn = registry[ref_cls]
      break
  return ldj_ratio_fn

def forward_log_det_jacobian_ratio(
    p,
    x,
    q,
    y,
    event_ndims,
    p_kwargs=None,
    q_kwargs=None,
):
  """Computes `p.fldj(x, ndims) - q.fdlj(y, ndims)`, numerically stably.

  `p_kwargs` and `q_kwargs` are passed to the registered `fldj_ratio_fn`. The
  fallback implementation passes them to the `forward_log_det_jacobian` methods
  of `p` and `q`.

  Args:
    p: A bijector instance.
    x: A tensor from the preimage of `p.forward`.
    q: A bijector instance of the same type as `p`, with matching shape.
    y: A tensor from the preimage of `q.forward`.
    event_ndims: The number of right-hand dimensions comprising the event shapes
      of `x` and `y`.
    p_kwargs: Keyword args to pass to `p`.
    q_kwargs: Keyword args to pass to `q`.

  Returns:
    fldj_ratio: `log ((abs o det o jac p)(x) / (abs o det o jac q)(y))`,
      i.e. in TFP code, `p.forward_log_det_jacobian(x, event_ndims) -
      q.forward_log_det_jacobian(y, event_ndims)`. In some cases
      this will be computed with better than naive numerical precision, e.g. by
      moving differences inside of a sum reduction.
  """
  assert type(p) == type(q)  # pylint: disable=unidiomatic-typecheck
  if p_kwargs is None:
    p_kwargs = {}
  if q_kwargs is None:
    q_kwargs = {}

  fldj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_fldj_ratio_registry)
  ildj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_ildj_ratio_registry)

  def inverse_fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
    # p.fldj(x) - q.fldj(y) = q.ildj(q(y)) - p.ildj(p(x))
    return ildj_ratio_fn(q, q.forward(y), p, p.forward(x),
                         q.forward_event_ndims(event_ndims), q_kwargs, p_kwargs)

  if fldj_ratio_fn is None:
    if ildj_ratio_fn is None:
      fldj_ratio_fn = _default_fldj_ratio_fn
    else:
      fldj_ratio_fn = inverse_fldj_ratio_fn

  return fldj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs)

def inverse_log_det_jacobian_ratio(
    p,
    x,
    q,
    y,
    event_ndims,
    p_kwargs=None,
    q_kwargs=None,
):
  """Computes `p.ildj(x, ndims) - q.idlj(y, ndims)`, numerically stably.

  `p_kwargs` and `q_kwargs` are passed to the registered `ildj_ratio_fn`. The
  fallback implementation passes them to the `inverse_log_det_jacobian` methods
  of `p` and `q`.

  Args:
    p: A bijector instance.
    x: A tensor from the image of `p.forward`.
    q: A bijector instance of the same type as `p`, with matching shape.
    y: A tensor from the image of `q.forward`.
    event_ndims: The number of right-hand dimensions comprising the event shapes
      of `x` and `y`.
    p_kwargs: Keyword args to pass to `p`.
    q_kwargs: Keyword args to pass to `q`.

  Returns:
    ildj_ratio: `log ((abs o det o jac p^-1)(x) / (abs o det o jac q^-1)(y))`,
      i.e. in TFP code, `p.inverse_log_det_jacobian(x, event_ndims) -
      q.inverse_log_det_jacobian(y, event_ndims)`. In some cases
      this will be computed with better than naive numerical precision, e.g. by
      moving differences inside of a sum reduction.
  """
  assert type(p) == type(q)  # pylint: disable=unidiomatic-typecheck
  if p_kwargs is None:
    p_kwargs = {}
  if q_kwargs is None:
    q_kwargs = {}

  ildj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_ildj_ratio_registry)
  fldj_ratio_fn = _get_ldj_ratio_fn(type(p), registry=_fldj_ratio_registry)

  def inverse_ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs):
    # p.ildj(x) - q.ildj(y) = q.fldj(q^-1(y)) - p.fldj(p^-1(x))
    return fldj_ratio_fn(q, q.inverse(y), p, p.inverse(x),
                         q.inverse_event_ndims(event_ndims), q_kwargs, p_kwargs)

  if ildj_ratio_fn is None:
    if fldj_ratio_fn is None:
      ildj_ratio_fn = _default_ildj_ratio_fn
    else:
      ildj_ratio_fn = inverse_ildj_ratio_fn

  return ildj_ratio_fn(p, x, q, y, event_ndims, p_kwargs, q_kwargs)

class RegisterFLDJRatio(object):

  def __init__(self, bijector_class):
    self.cls = bijector_class

  def __call__(self, fn):
    assert self.cls not in _fldj_ratio_registry
    _fldj_ratio_registry[self.cls] = fn
    return fn

class RegisterILDJRatio(object):

  def __init__(self, bijector_class):
    self.cls = bijector_class

  def __call__(self, fn):
    assert self.cls not in _ildj_ratio_registry
    _ildj_ratio_registry[self.cls] = fn
    return fn
<|endoftext|>


<commit_before>import uuid
from datetime import datetime
from unittest.mock import call

import pytest
from flask import json

from app.dao.services_dao import dao_fetch_services_by_sms_sender
from app.notifications.receive_notifications import (
    format_mmg_message,
    format_mmg_datetime,
    create_inbound_sms_object,
    strip_leading_forty_four
)

from app.models import InboundSms, EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE
from tests.app.db import create_service
from tests.app.conftest import sample_service

def test_receive_notification_returns_received_to_mmg(client, mocker, sample_service_full_permissions):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    data = {
        "ID": "1234",
        "MSISDN": "447700900855",
        "Message": "Some message to notify",
        "Trigger": "Trigger?",
        "Number": "testing",
        "Channel": "SMS",
        "DateRecieved": "2012-06-27 12:33:00"
    }
    response = client.post(path='/notifications/sms/receive/mmg',
                           data=json.dumps(data),
                           headers=[('Content-Type', 'application/json')])

    assert response.status_code == 200
    assert response.get_data(as_text=True) == 'RECEIVED'
    inbound_sms_id = InboundSms.query.all()[0].id
    mocked.assert_called_once_with(
        [str(inbound_sms_id), str(sample_service_full_permissions.id)], queue="notify-internal-tasks")

@pytest.mark.parametrize('provider,headers,data,expected_response', [
    (
        'mmg',
        [('Content-Type', 'application/json')],
        json.dumps({
            "ID": "1234",
            "MSISDN": "447700900855",
            "Message": "Some message to notify",
            "Trigger": "Trigger?",
            "Number": "testing",
            "Channel": "SMS",
            "DateRecieved": "2012-06-27 12:33:00"
        }),
        'RECEIVED'
    ),
    (
        'firetext',
        None,
        {
            "Message": "Some message to notify",
            "source": "Source",
            "time": "2012-06-27 12:33:00",
            "destination": "447700900855"
        },
        '{\n  "status": "ok"\n}'
    ),
])
@pytest.mark.parametrize('permissions', [
    ([SMS_TYPE]),
    ([INBOUND_SMS_TYPE]),
])
def test_receive_notification_without_permissions_does_not_create_inbound(
        client, mocker, notify_db, notify_db_session, permissions, provider, headers, data, expected_response):
    service = sample_service(notify_db, notify_db_session, permissions=permissions)
    mocker.patch("app.notifications.receive_notifications.dao_fetch_services_by_sms_sender",
                 return_value=[service])
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mocked_logger = mocker.patch("flask.current_app.logger.error")

    response = client.post(path='/notifications/sms/receive/{}'.format(provider),
                           data=data,
                           headers=headers)

    assert response.status_code == 200
    assert response.get_data(as_text=True) == expected_response
    assert len(InboundSms.query.all()) == 0
    assert mocked.called is False
    assert mocked_logger.call_args == call('Service "{}" does not allow inbound SMS'.format(service.id))

@pytest.mark.parametrize('message, expected_output', [
    ('abc', 'abc'),
    ('', ''),
    ('lots+of+words', 'lots of words'),
    ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'),
    ('x+%2B+y', 'x + y')
])
def test_format_mmg_message(message, expected_output):
    assert format_mmg_message(message) == expected_output

@pytest.mark.parametrize('provider_date, expected_output', [
    ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)),
    ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 10, 56, 11))
])
def test_format_mmg_datetime(provider_date, expected_output):
    assert format_mmg_datetime(provider_date) == expected_output

def test_create_inbound_mmg_sms_object(sample_service_full_permissions):
    sample_service_full_permissions.sms_sender = 'foo'
    data = {
        'Message': 'hello+there+%F0%9F%93%A9',
        'Number': 'foo',
        'MSISDN': '07700 900 001',
        'DateRecieved': '2017-01-02+03%3A04%3A05',
        'ID': 'bar',
    }

    inbound_sms = create_inbound_sms_object(sample_service_full_permissions, format_mmg_message(data["Message"]),
                                            data["MSISDN"], data["ID"], data["DateRecieved"], "mmg")

    assert inbound_sms.service_id == sample_service_full_permissions.id
    assert inbound_sms.notify_number == 'foo'
    assert inbound_sms.user_number == '447700900001'
    assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5)
    assert inbound_sms.provider_reference == 'bar'
    assert inbound_sms._content != 'hello there 📩'
    assert inbound_sms.content == 'hello there 📩'
    assert inbound_sms.provider == 'mmg'

@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services'])
def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number):
    create_service(service_name='a', sms_sender='foo', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
    create_service(service_name='b', sms_sender='foo', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = {
        'Message': 'hello',
        'Number': notify_number,
        'MSISDN': '7700900001',
        'DateRecieved': '2017-01-02 03:04:05',
        'ID': 'bar',
    }
    response = client.post(path='/notifications/sms/receive/mmg',
                           data=json.dumps(data),
                           headers=[('Content-Type', 'application/json')])

    # we still return 'RECEIVED' to MMG
    assert response.status_code == 200
    assert response.get_data(as_text=True) == 'RECEIVED'
    assert InboundSms.query.count() == 0

def test_receive_notification_returns_received_to_firetext(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    service = create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    mock.assert_has_calls([call('inbound.firetext.successful')])

    assert result['status'] == 'ok'
    inbound_sms_id = InboundSms.query.all()[0].id
    mocked.assert_called_once_with([str(inbound_sms_id), str(service.id)], queue="notify-internal-tasks")

def test_receive_notification_from_firetext_persists_message(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    service = create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    persisted = InboundSms.query.first()

    assert result['status'] == 'ok'
    assert persisted.notify_number == '07111111111'
    assert persisted.user_number == '447999999999'
    assert persisted.service == service
    assert persisted.content == 'this is a message'
    assert persisted.provider == 'firetext'
    assert persisted.provider_date == datetime(2017, 1, 1, 12, 0, 0, 0)
    mocked.assert_called_once_with([str(persisted.id), str(service.id)], queue="notify-internal-tasks")

def test_receive_notification_from_firetext_persists_message_with_normalized_phone(notify_db_session, client, mocker):
    mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    persisted = InboundSms.query.first()

    assert result['status'] == 'ok'
    assert persisted.user_number == '447999999999'

def test_returns_ok_to_firetext_if_mismatched_sms_sender(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    create_service(
        service_name='b', sms_sender='07111111199', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    assert not InboundSms.query.all()
    assert result['status'] == 'ok'
    mock.assert_has_calls([call('inbound.firetext.failed')])
    mocked.call_count == 0

@pytest.mark.parametrize(
    'number, expected',
    [
        ('447123123123', '07123123123'),
        ('447123123144', '07123123144'),
        ('07123123123', '07123123123'),
        ('447444444444', '07444444444')
    ]
)
def test_strip_leading_country_code(number, expected):
    assert strip_leading_forty_four(number) == expected
<commit_msg>Refactored test<commit_after>import uuid
from datetime import datetime
from unittest.mock import call

import pytest
from flask import json

from app.dao.services_dao import dao_fetch_services_by_sms_sender
from app.notifications.receive_notifications import (
    format_mmg_message,
    format_mmg_datetime,
    create_inbound_sms_object,
    strip_leading_forty_four
)

from app.models import InboundSms, EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE
from tests.app.db import create_service
from tests.app.conftest import sample_service

def test_receive_notification_returns_received_to_mmg(client, mocker, sample_service_full_permissions):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    data = {
        "ID": "1234",
        "MSISDN": "447700900855",
        "Message": "Some message to notify",
        "Trigger": "Trigger?",
        "Number": "testing",
        "Channel": "SMS",
        "DateRecieved": "2012-06-27 12:33:00"
    }
    response = client.post(path='/notifications/sms/receive/mmg',
                           data=json.dumps(data),
                           headers=[('Content-Type', 'application/json')])

    assert response.status_code == 200
    assert response.get_data(as_text=True) == 'RECEIVED'
    inbound_sms_id = InboundSms.query.all()[0].id
    mocked.assert_called_once_with(
        [str(inbound_sms_id), str(sample_service_full_permissions.id)], queue="notify-internal-tasks")

@pytest.mark.parametrize('provider,headers,data,expected_response', [
    (
        'mmg',
        [('Content-Type', 'application/json')],
        json.dumps({
            "ID": "1234",
            "MSISDN": "447700900855",
            "Message": "Some message to notify",
            "Trigger": "Trigger?",
            "Number": "testing",
            "Channel": "SMS",
            "DateRecieved": "2012-06-27 12:33:00"
        }),
        'RECEIVED'
    ),
    (
        'firetext',
        None,
        {
            "Message": "Some message to notify",
            "source": "Source",
            "time": "2012-06-27 12:33:00",
            "destination": "447700900855"
        },
        '{\n  "status": "ok"\n}'
    ),
])
@pytest.mark.parametrize('permissions', [
    ([SMS_TYPE]),
    ([INBOUND_SMS_TYPE]),
])
def test_receive_notification_without_permissions_does_not_create_inbound(
        client, mocker, notify_db, notify_db_session, permissions, provider, headers, data, expected_response):
    service = sample_service(notify_db, notify_db_session, permissions=permissions)
    mocker.patch("app.notifications.receive_notifications.dao_fetch_services_by_sms_sender",
                 return_value=[service])
    mocked_send_inbound_sms = mocker.patch(
        "app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mocked_logger = mocker.patch("flask.current_app.logger.error")

    response = client.post(path='/notifications/sms/receive/{}'.format(provider),
                           data=data,
                           headers=headers)

    assert response.status_code == 200
    assert response.get_data(as_text=True) == expected_response
    assert len(InboundSms.query.all()) == 0
    mocked_send_inbound_sms.assert_not_called()
    mocked_logger.assert_called_once_with('Service "{}" does not allow inbound SMS'.format(service.id))

@pytest.mark.parametrize('message, expected_output', [
    ('abc', 'abc'),
    ('', ''),
    ('lots+of+words', 'lots of words'),
    ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'),
    ('x+%2B+y', 'x + y')
])
def test_format_mmg_message(message, expected_output):
    assert format_mmg_message(message) == expected_output

@pytest.mark.parametrize('provider_date, expected_output', [
    ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)),
    ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 10, 56, 11))
])
def test_format_mmg_datetime(provider_date, expected_output):
    assert format_mmg_datetime(provider_date) == expected_output

def test_create_inbound_mmg_sms_object(sample_service_full_permissions):
    sample_service_full_permissions.sms_sender = 'foo'
    data = {
        'Message': 'hello+there+%F0%9F%93%A9',
        'Number': 'foo',
        'MSISDN': '07700 900 001',
        'DateRecieved': '2017-01-02+03%3A04%3A05',
        'ID': 'bar',
    }

    inbound_sms = create_inbound_sms_object(sample_service_full_permissions, format_mmg_message(data["Message"]),
                                            data["MSISDN"], data["ID"], data["DateRecieved"], "mmg")

    assert inbound_sms.service_id == sample_service_full_permissions.id
    assert inbound_sms.notify_number == 'foo'
    assert inbound_sms.user_number == '447700900001'
    assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5)
    assert inbound_sms.provider_reference == 'bar'
    assert inbound_sms._content != 'hello there 📩'
    assert inbound_sms.content == 'hello there 📩'
    assert inbound_sms.provider == 'mmg'

@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services'])
def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number):
    create_service(service_name='a', sms_sender='foo', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
    create_service(service_name='b', sms_sender='foo', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = {
        'Message': 'hello',
        'Number': notify_number,
        'MSISDN': '7700900001',
        'DateRecieved': '2017-01-02 03:04:05',
        'ID': 'bar',
    }
    response = client.post(path='/notifications/sms/receive/mmg',
                           data=json.dumps(data),
                           headers=[('Content-Type', 'application/json')])

    # we still return 'RECEIVED' to MMG
    assert response.status_code == 200
    assert response.get_data(as_text=True) == 'RECEIVED'
    assert InboundSms.query.count() == 0

def test_receive_notification_returns_received_to_firetext(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    service = create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    mock.assert_has_calls([call('inbound.firetext.successful')])

    assert result['status'] == 'ok'
    inbound_sms_id = InboundSms.query.all()[0].id
    mocked.assert_called_once_with([str(inbound_sms_id), str(service.id)], queue="notify-internal-tasks")

def test_receive_notification_from_firetext_persists_message(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    service = create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    persisted = InboundSms.query.first()

    assert result['status'] == 'ok'
    assert persisted.notify_number == '07111111111'
    assert persisted.user_number == '447999999999'
    assert persisted.service == service
    assert persisted.content == 'this is a message'
    assert persisted.provider == 'firetext'
    assert persisted.provider_date == datetime(2017, 1, 1, 12, 0, 0, 0)
    mocked.assert_called_once_with([str(persisted.id), str(service.id)], queue="notify-internal-tasks")

def test_receive_notification_from_firetext_persists_message_with_normalized_phone(notify_db_session, client, mocker):
    mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    create_service(
        service_name='b', sms_sender='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    persisted = InboundSms.query.first()

    assert result['status'] == 'ok'
    assert persisted.user_number == '447999999999'

def test_returns_ok_to_firetext_if_mismatched_sms_sender(notify_db_session, client, mocker):
    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')

    create_service(
        service_name='b', sms_sender='07111111199', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])

    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"

    response = client.post(
        path='/notifications/sms/receive/firetext',
        data=data,
        headers=[('Content-Type', 'application/x-www-form-urlencoded')])

    assert response.status_code == 200
    result = json.loads(response.get_data(as_text=True))

    assert not InboundSms.query.all()
    assert result['status'] == 'ok'
    mock.assert_has_calls([call('inbound.firetext.failed')])
    mocked.call_count == 0

@pytest.mark.parametrize(
    'number, expected',
    [
        ('447123123123', '07123123123'),
        ('447123123144', '07123123144'),
        ('07123123123', '07123123123'),
        ('447444444444', '07444444444')
    ]
)
def test_strip_leading_country_code(number, expected):
    assert strip_leading_forty_four(number) == expected
<|endoftext|>


<commit_before>import unittest
import warnings

import chainer
from chainer import testing
from chainer import utils

def f():
    utils.experimental('test.f')

def g():
    utils.experimental()

def h(x):
    utils.experimental()

class C(object):

    @staticmethod
    def static_method():
        utils.experimental()

    @classmethod
    def class_method(cls):
        utils.experimental()

    def __init__(self):
        utils.experimental()

    def f(self):
        utils.experimental()

class TestExperimental(unittest.TestCase):

    def setUp(self):
        self.original = chainer.disable_experimental_feature_warning
        chainer.disable_experimental_feature_warning = False

    def tearDown(self):
        chainer.disable_experimental_feature_warning = self.original

    def test_experimental_with_api_name(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            f()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('test.f is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            g()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('g is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name_2(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name_2(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            c = C()

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            c.f()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C.f is an experimental API.', str(w[0].message))

    def test_experimental_static_method(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C.static_method()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('static_method is an experimental API.',
                      str(w[0].message))

    def test_experimental_class_method(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C.class_method()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C.class_method is an experimental API.',
                      str(w[0].message))

class TestDisableExperimentalWarning(unittest.TestCase):

    def setUp(self):
        self.original = chainer.disable_experimental_feature_warning
        chainer.disable_experimental_feature_warning = True

    def tearDown(self):
        chainer.disable_experimental_feature_warning = self.original

    def test_experimental(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            f()

        self.assertEqual(len(w), 0)

testing.run_module(__name__, __file__)
<commit_msg>Fix test name<commit_after>import unittest
import warnings

import chainer
from chainer import testing
from chainer import utils

def f():
    utils.experimental('test.f')

def g():
    utils.experimental()

def h(x):
    utils.experimental()

class C(object):

    @staticmethod
    def static_method():
        utils.experimental()

    @classmethod
    def class_method(cls):
        utils.experimental()

    def __init__(self):
        utils.experimental()

    def f(self):
        utils.experimental()

class TestExperimental(unittest.TestCase):

    def setUp(self):
        self.original = chainer.disable_experimental_feature_warning
        chainer.disable_experimental_feature_warning = False

    def tearDown(self):
        chainer.disable_experimental_feature_warning = self.original

    def test_experimental_with_api_name(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            f()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('test.f is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            g()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('g is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name_2(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C is an experimental API.', str(w[0].message))

    def test_experimental_with_no_api_name_3(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            c = C()

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            c.f()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C.f is an experimental API.', str(w[0].message))

    def test_experimental_static_method(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C.static_method()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('static_method is an experimental API.',
                      str(w[0].message))

    def test_experimental_class_method(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            C.class_method()

        self.assertEqual(len(w), 1)
        self.assertIs(w[0].category, FutureWarning)
        self.assertIn('C.class_method is an experimental API.',
                      str(w[0].message))

class TestDisableExperimentalWarning(unittest.TestCase):

    def setUp(self):
        self.original = chainer.disable_experimental_feature_warning
        chainer.disable_experimental_feature_warning = True

    def tearDown(self):
        chainer.disable_experimental_feature_warning = self.original

    def test_experimental(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            f()

        self.assertEqual(len(w), 0)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import os
import queue
import re
import signal
from subprocess import CalledProcessError
from subprocess import PIPE
from subprocess import Popen
import threading
import unittest

import numpy as np
import pytest

import chainer.testing
import chainer.testing.attr
import chainermn
from chainermn.communicators import _memory_utility

class _TimeoutThread(threading.Thread):
    def __init__(self, queue, rank):
        super(_TimeoutThread, self).__init__()
        self.queue = queue
        self.rank = rank

    def run(self):
        try:
            self.queue.get(timeout=60)
        except queue.Empty:
            # Show error message and information of the problem
            try:
                p = Popen(['ompi_info', '--all', '--parsable'], stdout=PIPE)
                out, err = p.communicate()
                if type(out) == bytes:
                    out = out.decode('utf-8')
                m = re.search(r'ompi:version:full:(\S+)', out)
                version = m.group(1)

                msg = "\n\n***** ERROR:" \
                      "It looks like you are using " \
                      "Open MPI version {}." \
                      "It is known that the following Open MPI versions " \
                      "have a bug that cause MPI_Bcast() deadlock when " \
                      "GPUDirect is used: " \
                      "3.0.0, 3.0.1, 3.0.2, 3.1.0, 3.1.1, 3.1.2\n"
                if self.rank == 1:
                    # Rank 1 prints the error message,
                    # Rank 0 may finish Bcast() immediately without deadlock,
                    # depending on the timing,
                    # because rank 0 is the root of Bcast().
                    print(msg.format(version), flush=True)

                os.kill(os.getpid(), signal.SIGKILL)
            except CalledProcessError:
                pass

class TestBcastDeadlock(unittest.TestCase):
    def setup(self, gpu):
        if gpu:
            self.communicator = chainermn.create_communicator('flat')
            self.device = self.communicator.intra_rank
            chainer.cuda.get_device_from_id(self.device).use()
        else:
            self.device = -1

        if self.communicator.size < 2:
            pytest.skip('This test is for at least two processes')

        self.queue = queue.Queue(maxsize=1)

    def teardown(self):
        pass

    @chainer.testing.attr.gpu
    def test_bcast_gpu_large_buffer_deadlock(self):
        """Regression test of Open MPI's issue #3972"""
        self.setup(True)
        buf_size = 10000
        mpi_comm = self.communicator.mpi_comm

        if self.communicator.rank == 0:
            array = np.arange(buf_size, dtype=np.float32)
        else:
            array = np.empty(buf_size, dtype=np.float32)

        array = chainer.cuda.to_gpu(array, device=self.device)

        ptr = _memory_utility.array_to_buffer_object(array)

        # This Bcast() cause deadlock if the underlying MPI has the bug.
        th = _TimeoutThread(self.queue, self.communicator.rank)
        th.start()
        mpi_comm.Bcast(ptr, root=0)
        mpi_comm.barrier()
        self.queue.put(True)
        assert True

        self.teardown()
<commit_msg>fix python2<commit_after>import os
try:
    import queue
except ImportError:
    import Queue as queue
import re
import signal
from subprocess import CalledProcessError
from subprocess import PIPE
from subprocess import Popen
import sys
import threading
import unittest

import numpy as np
import pytest

import chainer.testing
import chainer.testing.attr
import chainermn
from chainermn.communicators import _memory_utility

class _TimeoutThread(threading.Thread):
    def __init__(self, queue, rank):
        super(_TimeoutThread, self).__init__()
        self.queue = queue
        self.rank = rank

    def run(self):
        try:
            self.queue.get(timeout=60)
        except queue.Empty:
            # Show error message and information of the problem
            try:
                p = Popen(['ompi_info', '--all', '--parsable'], stdout=PIPE)
                out, err = p.communicate()
                if type(out) == bytes:
                    out = out.decode('utf-8')
                m = re.search(r'ompi:version:full:(\S+)', out)
                version = m.group(1)

                msg = "\n\n***** ERROR:" \
                      "It looks like you are using " \
                      "Open MPI version {}." \
                      "It is known that the following Open MPI versions " \
                      "have a bug that cause MPI_Bcast() deadlock when " \
                      "GPUDirect is used: " \
                      "3.0.0, 3.0.1, 3.0.2, 3.1.0, 3.1.1, 3.1.2\n"
                if self.rank == 1:
                    # Rank 1 prints the error message,
                    # Rank 0 may finish Bcast() immediately without deadlock,
                    # depending on the timing,
                    # because rank 0 is the root of Bcast().
                    print(msg.format(version))
                    sys.stdout.flush()

                os.kill(os.getpid(), signal.SIGKILL)
            except CalledProcessError:
                pass

class TestBcastDeadlock(unittest.TestCase):
    def setup(self, gpu):
        if gpu:
            self.communicator = chainermn.create_communicator('flat')
            self.device = self.communicator.intra_rank
            chainer.cuda.get_device_from_id(self.device).use()
        else:
            self.device = -1

        if self.communicator.size < 2:
            pytest.skip('This test is for at least two processes')

        self.queue = queue.Queue(maxsize=1)

    def teardown(self):
        pass

    @chainer.testing.attr.gpu
    def test_bcast_gpu_large_buffer_deadlock(self):
        """Regression test of Open MPI's issue #3972"""
        self.setup(True)
        buf_size = 10000
        mpi_comm = self.communicator.mpi_comm

        if self.communicator.rank == 0:
            array = np.arange(buf_size, dtype=np.float32)
        else:
            array = np.empty(buf_size, dtype=np.float32)

        array = chainer.cuda.to_gpu(array, device=self.device)

        ptr = _memory_utility.array_to_buffer_object(array)

        # This Bcast() cause deadlock if the underlying MPI has the bug.
        th = _TimeoutThread(self.queue, self.communicator.rank)
        th.start()
        mpi_comm.Bcast(ptr, root=0)
        mpi_comm.barrier()
        self.queue.put(True)
        assert True

        self.teardown()
<|endoftext|>


<commit_before>from __future__ import with_statement
import itertools
import unittest

import mpi4py.MPI
import numpy as np
import pytest

from chainer import testing
import chainermn
from chainermn.communicators.flat_communicator import FlatCommunicator
from chainermn.communicators.naive_communicator import NaiveCommunicator
import chainerx as chx

class TestDataset(unittest.TestCase):

    def setUp(self):
        self.mpi_comm = mpi4py.MPI.COMM_WORLD
        self.communicator = NaiveCommunicator(self.mpi_comm)

    def check_scatter_dataset(self, original_dataset, shuffle=False, root=0):
        if self.communicator.rank != root:
            original_dataset = None
        my_dataset = chainermn.scatter_dataset(
            original_dataset, self.communicator,
            shuffle=shuffle, root=root)
        sub_datasets = self.communicator.gather_obj(my_dataset, root=root)

        if self.communicator.rank == root:
            # Test the sizes
            sub_sizes = [len(sub_dataset) for sub_dataset in sub_datasets]
            self.assertEqual(len(set(sub_sizes)), 1)
            sub_size = sub_sizes[0]
            self.assertLessEqual(
                len(original_dataset), sub_size * self.mpi_comm.size)
            self.assertGreater(
                len(original_dataset), (sub_size - 1) * self.mpi_comm.size)

            # Test the content of scattered datasets
            joined_dataset = sum((sub_dataset[:]
                                  for sub_dataset in sub_datasets), [])

            # NOTE: values in original_dataset must be int-like type
            # (i.e. dtype('int64') or dtype('int32')
            # This is necessary because numpy and cupy/chainerx have
            # different behaviours on this.
            # type(numpy.array([1])[0])
            # =>  <class 'numpy.int64'>  # Scalar
            # type(chainerx.array([1])[0])
            # => <class 'chainerx.ndarray'>  # array of one element
            joined_dataset = [int(e) for e in joined_dataset]
            original_dataset = [int(e) for e in original_dataset]
            self.assertEqual(set(joined_dataset), set(original_dataset))

    def test_scatter_dataset(self):
        n = self.communicator.size

        for shuffle in [True, False]:
            for root in range(self.communicator.size):
                self.communicator.mpi_comm.barrier()
                self.check_scatter_dataset([], shuffle, root)
                self.check_scatter_dataset([0], shuffle, root)
                self.check_scatter_dataset(list(range(n)), shuffle, root)
                self.check_scatter_dataset(list(range(n * 5 - 1)),
                                           shuffle, root)

                self.check_scatter_dataset(np.array([]), shuffle, root)
                self.check_scatter_dataset(np.array([0]), shuffle, root)
                self.check_scatter_dataset(np.arange(n), shuffle, root)
                self.check_scatter_dataset(np.arange(n * 5 - 1), shuffle, root)

                self.check_scatter_dataset(chx.array([]), shuffle, root)
                self.check_scatter_dataset(chx.array([0]), shuffle, root)
                self.check_scatter_dataset(chx.arange(n), shuffle, root)
                self.check_scatter_dataset(
                    chx.arange(n * 5 - 1), shuffle, root)

def scatter_large_data(communicator):
    data = []
    if communicator.rank == 0:
        data = ['test'] * 2000000000
    data = chainermn.scatter_dataset(data, communicator)
    assert len(data) > 0

@testing.attr.slow
def test_scatter_large_dataset_naive():
    mpi_comm = mpi4py.MPI.COMM_WORLD
    communicator = NaiveCommunicator(mpi_comm)

    # This test only runs when comm.size >= 2.
    if communicator.size == 1:
        pytest.skip('This test is for multinode')

    scatter_large_data(communicator)

@testing.attr.gpu
@testing.attr.slow
def test_scatter_large_dataset_flat():
    mpi_comm = mpi4py.MPI.COMM_WORLD
    communicator = FlatCommunicator(mpi_comm)

    # This test only runs when comm.size >= 2.
    if communicator.size == 1:
        pytest.skip('This test is for multinode')

    scatter_large_data(communicator)

def test_scatter_index_one():
    it = chainermn.datasets.scatter._scatter_index(10, 3, False)
    split = [(0, 0, 4), (1, 4, 7), (2, 7, 10)]
    for lhs, rhs in zip(split, it):
        assert lhs == rhs

@pytest.mark.parametrize('combination', [
    [10, 3], [1244, 23], [2, 1], [230945, 237]])
def test_scatter_index(combination):
    length, size = combination
    it = chainermn.datasets.scatter._scatter_index(length, size, False)
    union = set()
    total = []
    subsets = []
    for (_, b, e) in it:
        subset = list(range(b, e))
        subsets.append(subset)
        total.extend(subset)
        for x in subset:
            union.add(x)
    assert length == len(total)  # no duplication
    assert length == len(union)  # no duplication & no lacking
    for lhs, rhs in itertools.combinations(subsets, 2):
        set(lhs).isdisjoint(set(rhs))
        assert abs(len(lhs) - len(rhs)) <= 1
<commit_msg>Remove unnecessary debug code.<commit_after>from __future__ import with_statement
import itertools
import unittest

import mpi4py.MPI
import numpy as np
import pytest

from chainer import testing
import chainermn
from chainermn.communicators.flat_communicator import FlatCommunicator
from chainermn.communicators.naive_communicator import NaiveCommunicator
import chainerx as chx

class TestDataset(unittest.TestCase):

    def setUp(self):
        self.mpi_comm = mpi4py.MPI.COMM_WORLD
        self.communicator = NaiveCommunicator(self.mpi_comm)

    def check_scatter_dataset(self, original_dataset, shuffle=False, root=0):
        if self.communicator.rank != root:
            original_dataset = None
        my_dataset = chainermn.scatter_dataset(
            original_dataset, self.communicator,
            shuffle=shuffle, root=root)
        sub_datasets = self.communicator.gather_obj(my_dataset, root=root)

        if self.communicator.rank == root:
            # Test the sizes
            sub_sizes = [len(sub_dataset) for sub_dataset in sub_datasets]
            self.assertEqual(len(set(sub_sizes)), 1)
            sub_size = sub_sizes[0]
            self.assertLessEqual(
                len(original_dataset), sub_size * self.mpi_comm.size)
            self.assertGreater(
                len(original_dataset), (sub_size - 1) * self.mpi_comm.size)

            # Test the content of scattered datasets
            joined_dataset = sum((sub_dataset[:]
                                  for sub_dataset in sub_datasets), [])

            # NOTE: values in original_dataset must be int-like type
            # (i.e. dtype('int64') or dtype('int32')
            # This is necessary because numpy and cupy/chainerx have
            # different behaviours on this.
            # type(numpy.array([1])[0])
            # =>  <class 'numpy.int64'>  # Scalar
            # type(chainerx.array([1])[0])
            # => <class 'chainerx.ndarray'>  # array of one element
            joined_dataset = [int(e) for e in joined_dataset]
            original_dataset = [int(e) for e in original_dataset]
            self.assertEqual(set(joined_dataset), set(original_dataset))

    def test_scatter_dataset(self):
        n = self.communicator.size

        for shuffle in [True, False]:
            for root in range(self.communicator.size):
                self.check_scatter_dataset([], shuffle, root)
                self.check_scatter_dataset([0], shuffle, root)
                self.check_scatter_dataset(list(range(n)), shuffle, root)
                self.check_scatter_dataset(list(range(n * 5 - 1)),
                                           shuffle, root)

                self.check_scatter_dataset(np.array([]), shuffle, root)
                self.check_scatter_dataset(np.array([0]), shuffle, root)
                self.check_scatter_dataset(np.arange(n), shuffle, root)
                self.check_scatter_dataset(np.arange(n * 5 - 1), shuffle, root)

                self.check_scatter_dataset(chx.array([]), shuffle, root)
                self.check_scatter_dataset(chx.array([0]), shuffle, root)
                self.check_scatter_dataset(chx.arange(n), shuffle, root)
                self.check_scatter_dataset(
                    chx.arange(n * 5 - 1), shuffle, root)

def scatter_large_data(communicator):
    data = []
    if communicator.rank == 0:
        data = ['test'] * 2000000000
    data = chainermn.scatter_dataset(data, communicator)
    assert len(data) > 0

@testing.attr.slow
def test_scatter_large_dataset_naive():
    mpi_comm = mpi4py.MPI.COMM_WORLD
    communicator = NaiveCommunicator(mpi_comm)

    # This test only runs when comm.size >= 2.
    if communicator.size == 1:
        pytest.skip('This test is for multinode')

    scatter_large_data(communicator)

@testing.attr.gpu
@testing.attr.slow
def test_scatter_large_dataset_flat():
    mpi_comm = mpi4py.MPI.COMM_WORLD
    communicator = FlatCommunicator(mpi_comm)

    # This test only runs when comm.size >= 2.
    if communicator.size == 1:
        pytest.skip('This test is for multinode')

    scatter_large_data(communicator)

def test_scatter_index_one():
    it = chainermn.datasets.scatter._scatter_index(10, 3, False)
    split = [(0, 0, 4), (1, 4, 7), (2, 7, 10)]
    for lhs, rhs in zip(split, it):
        assert lhs == rhs

@pytest.mark.parametrize('combination', [
    [10, 3], [1244, 23], [2, 1], [230945, 237]])
def test_scatter_index(combination):
    length, size = combination
    it = chainermn.datasets.scatter._scatter_index(length, size, False)
    union = set()
    total = []
    subsets = []
    for (_, b, e) in it:
        subset = list(range(b, e))
        subsets.append(subset)
        total.extend(subset)
        for x in subset:
            union.add(x)
    assert length == len(total)  # no duplication
    assert length == len(union)  # no duplication & no lacking
    for lhs, rhs in itertools.combinations(subsets, 2):
        set(lhs).isdisjoint(set(rhs))
        assert abs(len(lhs) - len(rhs)) <= 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Set up the path so we can import `remote`.
from _path import setPath
setPath(__file__)

from multiprocessing import Process
import logging
import SocketServer

from unittestwrapper import unittest

from remote.selectChannel.communication import SelectCommunicator
from remote.selectChannel.server import TCPHandler
from remote.control import Control

logger = logging.getLogger("remote.tests.queuedClientServer_test")

class TestQueuedClientServer(unittest.TestCase):
    def setUp(self):
        self.serverAddr = '127.0.0.1', 2695
        self.text = "This is my text. Isn't it beautiful?"

        def runServer():
            server = SocketServer.TCPServer(self.serverAddr, TCPHandler)
            server.serve_forever()

        # Start the server.
        self.server = Process(target=runServer)
        self.server.start()

        #XXX: HACK!
        # Wait a bit to hopefully ensure that the server has started and is listening. (we should probably replace this
        # with a queue or something, since that would also allow us to signal a graceful shutdown in the server.)
        import time
        time.sleep(0.2)
        #XXX: /HACK!

        try:
            # Connect to the server.
            self.control = Control(self.serverAddr)
            self.channel = self.control.controlChannel
            print "CHANNEL ASSIGNED:", self.channel

            #TODO: Control should be doing this!
            self.comm = SelectCommunicator()
            self.comm.registerChannel(self.channel)

        except:
            logger.exception("Failed to set up %s test!", self.__class__)
            self.server.terminate()

    def tearDown(self):
        # Shut down the server forcefully.
        self.server.terminate()

        if hasattr(self, 'channel'):
            del self.channel

        if hasattr(self, 'control'):
            del self.control

        #TODO: Control should be doing this!
        if hasattr(self, 'comm'):
            del self.comm

    def test_for_echo(self):
        def receiveResponse(sender, **kwargs):
            addr = kwargs['remoteAddress']
            self.assertEqual(addr, self.serverAddr)

            message = kwargs['message']
            logger.debug("Got response from %s:%d: %r", addr[0], addr[1], message)

            expectedMessage = "Hello, %s:%d! You said %r." % (addr[0], addr[1], self.text)
            self.assertEqual(message, expectedMessage)

            # Exit the loop, so we end the test.
            self.comm.exitLoop()

        self.channel.incomingPacket.connect(receiveResponse, self.channel)

        self.channel.send(self.text)

        # Start the Communicator loop.
        self.comm.loop()

if __name__ == '__main__':
    logging.basicConfig(level=0)

    # Build the suite of tests
    suite = unittest.TestLoader().loadTestsFromTestCase(TestQueuedClientServer)

    # Run the tests
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>Added colored logging to help differentiate between server and client logs.<commit_after>#!/usr/bin/env python

# Set up the path so we can import `remote`.
from _path import setPath
setPath(__file__)

from multiprocessing import Process
import logging
import SocketServer
import string
import sys

from unittestwrapper import unittest

from remote.selectChannel.communication import SelectCommunicator
from remote.selectChannel.server import TCPHandler
from remote.control import Control

logger = logging.getLogger("remote.tests.queuedClientServer_test")

class TestQueuedClientServer(unittest.TestCase):
    colorLogFmtTemplate = string.Template(
            '%(levelname)s'
            ':'
            '${location}'
            ':'
            '%(name)s'
            ':'
            '%(message)s'
            )
    consoleLogFmtTemplate = string.Template(
            '\033[38;5;16;48;5;${color}m${location}'
            '\033[0;1;38;5;59m:'
            '\033[0;1m%(levelname)s'
            '\033[38;5;59m:'
            '\033[38;5;${color}m%(name)s'
            '\033[38;5;59m: '
            '\033[0;38;5;${color}m%(message)s'
            '\033[m'
            )
    clientLogFmt = consoleLogFmtTemplate.safe_substitute(location='client')
    serverLogFmt = consoleLogFmtTemplate.safe_substitute(location='server')
    clientColorLogFmt = consoleLogFmtTemplate.safe_substitute(location='client', color=70)
    serverColorLogFmt = consoleLogFmtTemplate.safe_substitute(location='server', color=75)

    def setUp(self):
        self.serverAddr = '127.0.0.1', 2695
        self.text = "This is my text. Isn't it beautiful?"

        def runServer():
            # Configure server logging.
            for handler in logging.getLogger().handlers:
                if isinstance(handler, logging.StreamHandler) and handler.stream == sys.stderr:
                    handler.setFormatter(logging.Formatter(self.serverColorLogFmt))
                else:
                    handler.setFormatter(logging.Formatter(self.serverLogFmt))

            server = SocketServer.TCPServer(self.serverAddr, TCPHandler)
            server.serve_forever()

        # Start the server.
        self.server = Process(target=runServer)
        self.server.start()

        #XXX: HACK!
        # Wait a bit to hopefully ensure that the server has started and is listening. (we should probably replace this
        # with a queue or something, since that would also allow us to signal a graceful shutdown in the server.)
        import time
        time.sleep(0.2)
        #XXX: /HACK!

        # Configure client logging.
        for handler in logging.getLogger().handlers:
            if isinstance(handler, logging.StreamHandler) and handler.stream == sys.stderr:
                handler.setFormatter(logging.Formatter(self.clientColorLogFmt))
            else:
                handler.setFormatter(logging.Formatter(self.clientLogFmt))

        try:
            # Connect to the server.
            self.control = Control(self.serverAddr)
            self.channel = self.control.controlChannel
            print "CHANNEL ASSIGNED:", self.channel

            #TODO: Control should be doing this!
            self.comm = SelectCommunicator()
            self.comm.registerChannel(self.channel)

        except:
            logger.exception("Failed to set up %s test!", self.__class__)
            self.server.terminate()

    def tearDown(self):
        # Shut down the server forcefully.
        self.server.terminate()

        if hasattr(self, 'channel'):
            del self.channel

        if hasattr(self, 'control'):
            del self.control

        #TODO: Control should be doing this!
        if hasattr(self, 'comm'):
            del self.comm

    def test_for_echo(self):
        def receiveResponse(sender, **kwargs):
            addr = kwargs['remoteAddress']
            self.assertEqual(addr, self.serverAddr)

            message = kwargs['message']
            logger.debug("Got response from %s:%d: %r", addr[0], addr[1], message)

            expectedMessage = "Hello, %s:%d! You said %r." % (addr[0], addr[1], self.text)
            self.assertEqual(message, expectedMessage)

            # Exit the loop, so we end the test.
            self.comm.exitLoop()

        self.channel.incomingPacket.connect(receiveResponse, self.channel)

        self.channel.send(self.text)

        # Start the Communicator loop.
        self.comm.loop()

if __name__ == '__main__':
    logging.basicConfig(level=0)

    # Build the suite of tests
    suite = unittest.TestLoader().loadTestsFromTestCase(TestQueuedClientServer)

    # Run the tests
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>import flask

from donut.testing.fixtures import client
import donut

def test_marketplace_home(client):
    donut.init('test')
    donut.before_request()
    rv = client.get(flask.url_for('marketplace.marketplace'))
    donut.teardown_request(None)

    assert rv.status_code == 200

def test_marketplace_category(client):
    donut.init('test')
    donut.before_request()
    rv = client.get(flask.url_for('marketplace.category', cat=1))
    #assert rv.status_code == 200

    rv = client.get(flask.url_for('marketplace.category', cat=2))
    #assert rv.status_code == 200
    donut.teardown_request(None)

def test_marketplace_query(client):
    rv = client.get(flask.url_for('marketplace.query', cat=2, q='great'))
    #assert rv.status_code == 200

    rv = client.get(flask.url_for('marketplace.query'))
    #assert rv.status_code == 404
<commit_msg>Fixed tests<commit_after>import flask

from donut.testing.fixtures import client
import donut

def test_marketplace_home(client):
    donut.init('test')
    donut.before_request()
    rv = client.get(flask.url_for('marketplace.marketplace'))
    donut.teardown_request(None)

    assert rv.status_code == 200

def test_marketplace_category(client):
    donut.init('test')
    donut.before_request()
    rv = client.get('marketplace/view?cat=1')
    assert rv.status_code == 200

    rv = client.get('marketplace/view?cat=2')
    assert rv.status_code == 200
    donut.teardown_request(None)

def test_marketplace_query(client):
    rv = client.get(flask.url_for('marketplace.query', cat=2, q='great'))
    #assert rv.status_code == 200

    rv = client.get(flask.url_for('marketplace.query'))
    #assert rv.status_code == 404
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Network representation and basic operations
# Daniel Klein, 5/10/2012

import numpy as np
import scipy.sparse as sparse
import networkx as nx
import matplotlib.pyplot as plt

from Covariate import NodeCovariate, EdgeCovariate

class Network:
    def __init__(self, N = None):
        if N:
            self.N = N
            self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)
            self.names = np.array(['n_%d' % n for n in range(N)])
        else:
            self.N = 0
            self.network = None
            self.names = None

        # Offset intitially set to None so simpler offset-free model
        # code can be used by default
        self.offset = None

        # Maps from names to 1-D and 2-D arrays, respectively
        self.node_covariates = {}
        self.edge_covariates = {}

    def tocsr(self):
        if self.is_sparse():
            self.network = self.network.tocsr()
        else:
            print 'Attempting CSR conversion of a non-sparse network.'
            raise

    def new_node_covariate(self, name, as_int = False):
        self.node_covariates[name] = NodeCovariate(self.names)
        return self.node_covariates[name]

    def new_node_covariate_int(self, name):
        self.node_covariates[name] = NodeCovariate(self.names, dtype = np.int)
        return self.node_covariates[name]

    def new_edge_covariate(self, name):
        self.edge_covariates[name] = EdgeCovariate(self.names)
        return self.edge_covariates[name]

    def initialize_offset(self):
        self.offset = EdgeCovariate(self.names)
        return self.offset

    def subnetwork(self, inds):
        sub_N = len(inds)
        sub = Network()

        if self.is_sparse():
            self.tocsr()
        sub.network = self.network[inds][:,inds]
        sub.names = self.names[inds]
        sub.N = sub_N

        sub.node_covariates = {}
        for node_covariate in self.node_covariates:
            src = self.node_covariates[node_covariate]
            sub.new_node_covariate(node_covariate).from_existing(src, inds)
        sub.edge_covariates = {}
        for edge_covariate in self.edge_covariates:
            src = self.edge_covariates[edge_covariate]
            sub.new_edge_covariate(edge_covariate).from_existing(src, inds)

        if self.offset:
            sub.initialize_offset().from_existing(self.offset, inds)

        return sub

    # Syntactic sugar to make network generation look like object mutation
    def generate(self, model, **opts):
        self.network = model.generate(self, **opts)

    def network_from_file_gexf(self, path):
        in_network = nx.read_gexf(path)
        self.N = in_network.number_of_nodes()
        self.names = np.array(in_network.nodes())
        self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)

        name_to_index = {}
        for i, n in enumerate(self.names):
            name_to_index[n] = i
        for s, t in in_network.edges():
            self.network[name_to_index[s],name_to_index[t]] = True

    def network_from_edges(self, edges):
        # First pass over edges to determine names and number of nodes
        names = set()
        N = 0
        for n_1, n_2 in edges:
            if not n_1 in names:
                names.add(n_1)
                N += 1
            if not n_2 in names:
                names.add(n_2)
                N += 1

        # Process list of names and assign indices
        self.N = N
        self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)
        self.names = np.array(list(names))
        name_to_index = {}
        for i, n in enumerate(self.names):
            name_to_index[n] = i

        # Second pass over edges to populate network
        for n_1, n_2 in edges:
            self.network[name_to_index[n_1],name_to_index[n_2]] = True

    def nodes(self):
        return self.names

    def adjacency_matrix(self):
        if self.is_sparse():
            return self.network.todense()
        else:
            return self.network

    def is_sparse(self):
        return sparse.issparse(self.network)

    def sparse_adjacency_matrix(self):
        if self.is_sparse():
            return np.array(self.network)
        else:
            print 'Asked for sparse adjacency matrix of non-sparse network.'
            raise

    def offset_extremes(self):
        if not self.offset:
            self.initialize_offset()

        # (Separately) sort rows and columns of adjacency matrix by
        # increasing sum
        A = np.asarray(self.adjacency_matrix())
        r_ord = np.argsort(A.sum(1))
        c_ord = np.argsort(A.sum(0))
        A = A[r_ord][:,c_ord]

        # Convenience function to set blocks of the offset
        def set_offset_block(r, c, val):
            self.offset[r_ord[r],c_ord[c]] = val

        # Recursively examine for submatrices that will send
        # corresponding EMLE parameter estimates to infinity
        to_screen = [(np.arange(self.N), np.arange(self.N))]
        while len(to_screen) > 0:
            r_act, c_act = to_screen.pop()

            A_act = A[r_act][:,c_act]
            n_act = A_act.shape
            trivial = [(0,0), (0,n_act[1]), (n_act[0],0), (n_act[0],n_act[1])]
            for (i,j) in [(i,j)
                         for i in range(n_act[0] + 1)
                         for j in range(n_act[1] + 1)]:
                if (i,j) in trivial: continue
                if np.any(A_act[:i][:,:j]): continue
                if not np.all(A_act[i:][:,j:]): continue
                set_offset_block(r_act[:i], c_act[:j], -np.inf)
                set_offset_block(r_act[i:], c_act[j:], np.inf)
                if i > 0 and j < n_act[1]:
                    A_sub = A_act[:i][:,j:]
                    if not np.any(A_sub):
                        set_offset_block(r_act[:i], c_act[j:], -np.inf)
                    elif np.all(A_sub):
                        set_offset_block(r_act[:i], c_act[j:], np.inf)
                    else:
                        to_screen.append((r_act[np.arange(i)],
                                          c_act[np.arange(j, n_act[1])]))
                if i < n_act[0] and j > 0:
                    A_sub = A_act[i:][:,:j]
                    if not np.any(A_sub):
                        set_offset_block(r_act[i:], c_act[:j], -np.inf)
                    elif np.all(A_sub):
                        set_offset_block(r_act[i:], c_act[:j], np.inf)
                    else:
                        to_screen.append((r_act[np.arange(i, n_act[0])],
                                          c_act[np.arange(j)]))
                break

    def show(self):
        graph = nx.DiGraph()

        for n in self.nodes():
            graph.add_node(n)

        if self.is_sparse():
            nonzeros = set()
            nz_i, nz_j = self.network.nonzero()
            for n in range(self.network.nnz):
                graph.add_edge(self.names[nz_i[n]],self.names[nz_j[n]])
        else:
            for i in range(self.N):
                for j in range(self.N):
                    if self.network[i,j]:
                        graph.add_edge(self.names[i], self.names[j])

        nx.draw_graphviz(graph)
        plt.show()

    def show_heatmap(self, order_by = None):
        if order_by:
            title = 'Ordered by node covariate\n"%s"' % order_by
            o = np.argsort(self.node_covariates[order_by][:])
        else:
            title, o = 'Unordered', np.arange(self.N)

        f, (ax_im, ax_ord) = plt.subplots(2, sharex = True)
        f.set_figwidth(3)
        f.set_figheight(6)
        A = self.adjacency_matrix()
        ax_im.imshow(A[o][:,o]).set_cmap('binary')
        ax_im.set_ylim(0, self.N - 1)
        ax_im.set_yticks([])
        ax_im.set_title(title)
        plt.setp([ax_im.get_xticklabels(), ax_im.get_yticklabels()],
                 visible = False)
        if order_by:
            ax_ord.scatter(np.arange(self.N), self.node_covariates[order_by][o])
            ax_ord.set_xlim(0, self.N - 1)
            ax_ord.set_ylim(self.node_covariates[order_by][o[0]],
                            self.node_covariates[order_by][o[-1]])
        plt.show()

    def show_degree_histograms(self):
        # Messy since otherwise row/column sums can overflow...
        r = np.array(self.network.asfptype().sum(1),dtype=np.int).flatten()
        c = np.array(self.network.asfptype().sum(0),dtype=np.int).flatten()

        plt.figure()
        plt.subplot(2,1,1)
        plt.title('out-degree')
        plt.hist(r, bins = max(r))
        plt.subplot(2,1,2)
        plt.title('in-degree')
        plt.hist(c, bins = max(c))
        plt.show()

# Some "tests"
if __name__ == '__main__':
    net = Network()
    net.network_from_file_gexf('data/test.gexf')
    net.new_node_covariate('x_0')
    net.node_covariates['x_0'].from_pairs([str(i) for i in range(10)],
                                          [i**2 for i in range(10)])
    net.new_node_covariate('x_1')
    net.node_covariates['x_1'].data[:] = np.random.normal(2,1,net.N)
    def f_self(n_1, n_2):
        return n_1 == n_2
    net.new_edge_covariate('self_edge').from_binary_function_name(f_self)
    def f_first_half_dir(n_1, n_2):
        return (n_1 < n_2) and (n_2 in ['0','1','2','3','4'])
    net.new_edge_covariate('ec_2').from_binary_function_name(f_first_half_dir)

    print net.node_covariates['x_0']
    print net.node_covariates['x_1']
    print net.edge_covariates['self_edge']
    print net.edge_covariates['ec_2']

    print net.adjacency_matrix()
    print net.nodes()
    net.show()

    net_2 = net.subnetwork(np.array([5,0,1,6]))
    print net_2.adjacency_matrix()
    print net_2.node_covariates['x_0']
    print net_2.node_covariates['x_1']
    print net_2.edge_covariates['self_edge']
    print net_2.edge_covariates['ec_2']
    net_2.show()

    net_3 = Network(10)
    ord = np.arange(10)
    np.random.shuffle(ord)
    for i in range(10):
        for j in range(i,10):
            net_3.network[ord[i],ord[j]] = True
    net_3.offset_extremes()
    print net_3.offset.matrix()
<commit_msg>Added plotting of offset matrix.<commit_after>#!/usr/bin/env python

# Network representation and basic operations
# Daniel Klein, 5/10/2012

import numpy as np
import scipy.sparse as sparse
import networkx as nx
import matplotlib.pyplot as plt

from Covariate import NodeCovariate, EdgeCovariate

class Network:
    def __init__(self, N = None):
        if N:
            self.N = N
            self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)
            self.names = np.array(['n_%d' % n for n in range(N)])
        else:
            self.N = 0
            self.network = None
            self.names = None

        # Offset intitially set to None so simpler offset-free model
        # code can be used by default
        self.offset = None

        # Maps from names to 1-D and 2-D arrays, respectively
        self.node_covariates = {}
        self.edge_covariates = {}

    def tocsr(self):
        if self.is_sparse():
            self.network = self.network.tocsr()
        else:
            print 'Attempting CSR conversion of a non-sparse network.'
            raise

    def new_node_covariate(self, name, as_int = False):
        self.node_covariates[name] = NodeCovariate(self.names)
        return self.node_covariates[name]

    def new_node_covariate_int(self, name):
        self.node_covariates[name] = NodeCovariate(self.names, dtype = np.int)
        return self.node_covariates[name]

    def new_edge_covariate(self, name):
        self.edge_covariates[name] = EdgeCovariate(self.names)
        return self.edge_covariates[name]

    def initialize_offset(self):
        self.offset = EdgeCovariate(self.names)
        return self.offset

    def subnetwork(self, inds):
        sub_N = len(inds)
        sub = Network()

        if self.is_sparse():
            self.tocsr()
        sub.network = self.network[inds][:,inds]
        sub.names = self.names[inds]
        sub.N = sub_N

        sub.node_covariates = {}
        for node_covariate in self.node_covariates:
            src = self.node_covariates[node_covariate]
            sub.new_node_covariate(node_covariate).from_existing(src, inds)
        sub.edge_covariates = {}
        for edge_covariate in self.edge_covariates:
            src = self.edge_covariates[edge_covariate]
            sub.new_edge_covariate(edge_covariate).from_existing(src, inds)

        if self.offset:
            sub.initialize_offset().from_existing(self.offset, inds)

        return sub

    # Syntactic sugar to make network generation look like object mutation
    def generate(self, model, **opts):
        self.network = model.generate(self, **opts)

    def network_from_file_gexf(self, path):
        in_network = nx.read_gexf(path)
        self.N = in_network.number_of_nodes()
        self.names = np.array(in_network.nodes())
        self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)

        name_to_index = {}
        for i, n in enumerate(self.names):
            name_to_index[n] = i
        for s, t in in_network.edges():
            self.network[name_to_index[s],name_to_index[t]] = True

    def network_from_edges(self, edges):
        # First pass over edges to determine names and number of nodes
        names = set()
        N = 0
        for n_1, n_2 in edges:
            if not n_1 in names:
                names.add(n_1)
                N += 1
            if not n_2 in names:
                names.add(n_2)
                N += 1

        # Process list of names and assign indices
        self.N = N
        self.network = sparse.lil_matrix((self.N,self.N), dtype=np.bool)
        self.names = np.array(list(names))
        name_to_index = {}
        for i, n in enumerate(self.names):
            name_to_index[n] = i

        # Second pass over edges to populate network
        for n_1, n_2 in edges:
            self.network[name_to_index[n_1],name_to_index[n_2]] = True

    def nodes(self):
        return self.names

    def adjacency_matrix(self):
        if self.is_sparse():
            return self.network.todense()
        else:
            return self.network

    def is_sparse(self):
        return sparse.issparse(self.network)

    def sparse_adjacency_matrix(self):
        if self.is_sparse():
            return np.array(self.network)
        else:
            print 'Asked for sparse adjacency matrix of non-sparse network.'
            raise

    def offset_extremes(self):
        if not self.offset:
            self.initialize_offset()

        # (Separately) sort rows and columns of adjacency matrix by
        # increasing sum
        A = np.asarray(self.adjacency_matrix())
        r_ord = np.argsort(A.sum(1))
        c_ord = np.argsort(A.sum(0))
        A = A[r_ord][:,c_ord]

        # Convenience function to set blocks of the offset
        def set_offset_block(r, c, val):
            self.offset[r_ord[r],c_ord[c]] = val

        # Recursively examine for submatrices that will send
        # corresponding EMLE parameter estimates to infinity
        to_screen = [(np.arange(self.N), np.arange(self.N))]
        while len(to_screen) > 0:
            r_act, c_act = to_screen.pop()

            A_act = A[r_act][:,c_act]
            n_act = A_act.shape
            trivial = [(0,0), (0,n_act[1]), (n_act[0],0), (n_act[0],n_act[1])]
            for (i,j) in [(i,j)
                         for i in range(n_act[0] + 1)
                         for j in range(n_act[1] + 1)]:
                if (i,j) in trivial: continue
                if np.any(A_act[:i][:,:j]): continue
                if not np.all(A_act[i:][:,j:]): continue
                set_offset_block(r_act[:i], c_act[:j], -np.inf)
                set_offset_block(r_act[i:], c_act[j:], np.inf)
                if i > 0 and j < n_act[1]:
                    A_sub = A_act[:i][:,j:]
                    if not np.any(A_sub):
                        set_offset_block(r_act[:i], c_act[j:], -np.inf)
                    elif np.all(A_sub):
                        set_offset_block(r_act[:i], c_act[j:], np.inf)
                    else:
                        to_screen.append((r_act[np.arange(i)],
                                          c_act[np.arange(j, n_act[1])]))
                if i < n_act[0] and j > 0:
                    A_sub = A_act[i:][:,:j]
                    if not np.any(A_sub):
                        set_offset_block(r_act[i:], c_act[:j], -np.inf)
                    elif np.all(A_sub):
                        set_offset_block(r_act[i:], c_act[:j], np.inf)
                    else:
                        to_screen.append((r_act[np.arange(i, n_act[0])],
                                          c_act[np.arange(j)]))
                break

    def show(self):
        graph = nx.DiGraph()

        for n in self.nodes():
            graph.add_node(n)

        if self.is_sparse():
            nonzeros = set()
            nz_i, nz_j = self.network.nonzero()
            for n in range(self.network.nnz):
                graph.add_edge(self.names[nz_i[n]],self.names[nz_j[n]])
        else:
            for i in range(self.N):
                for j in range(self.N):
                    if self.network[i,j]:
                        graph.add_edge(self.names[i], self.names[j])

        nx.draw_graphviz(graph)
        plt.show()

    def show_heatmap(self, order_by = None):
        if order_by:
            title = 'Adjacency matrix ordered by node covariate\n"%s"' % order_by
            o = np.argsort(self.node_covariates[order_by][:])
        else:
            title, o = 'Unordered adjacency matrix', np.arange(self.N)

        f, (ax_im, ax_ord) = plt.subplots(2, sharex = True)
        f.set_figwidth(3)
        f.set_figheight(6)
        A = self.adjacency_matrix()
        ax_im.imshow(A[o][:,o]).set_cmap('binary')
        ax_im.set_ylim(0, self.N - 1)
        ax_im.set_yticks([])
        ax_im.set_title(title)
        plt.setp([ax_im.get_xticklabels(), ax_im.get_yticklabels()],
                 visible = False)
        if order_by:
            ax_ord.scatter(np.arange(self.N), self.node_covariates[order_by][o])
            ax_ord.set_xlim(0, self.N - 1)
            ax_ord.set_ylim(self.node_covariates[order_by][o[0]],
                            self.node_covariates[order_by][o[-1]])
        plt.show()

    def show_offset(self, order_by = None):
        if order_by:
            title = 'Offsets ordered by node covariate\n"%s"' % order_by
            o = np.argsort(self.node_covariates[order_by][:])
        else:
            title, o = 'Unordered offsets', np.arange(self.N)

        f = plt.figure()
        ax = f.add_subplot(1, 1, 1)
        O = self.offset.matrix()
        ax.imshow(O[o][:,o])
        ax.set_xlim(0, self.N - 1)
        ax.set_ylim(0, self.N - 1)
        ax.set_title(title)
        plt.setp([ax.get_xticklabels(), ax.get_yticklabels()],
                 visible = False)
        plt.show()

    def show_degree_histograms(self):
        # Messy since otherwise row/column sums can overflow...
        r = np.array(self.network.asfptype().sum(1),dtype=np.int).flatten()
        c = np.array(self.network.asfptype().sum(0),dtype=np.int).flatten()

        plt.figure()
        plt.subplot(2,1,1)
        plt.title('out-degree')
        plt.hist(r, bins = max(r))
        plt.subplot(2,1,2)
        plt.title('in-degree')
        plt.hist(c, bins = max(c))
        plt.show()

# Some "tests"
if __name__ == '__main__':
    net = Network()
    net.network_from_file_gexf('data/test.gexf')
    net.new_node_covariate('x_0')
    net.node_covariates['x_0'].from_pairs([str(i) for i in range(10)],
                                          [i**2 for i in range(10)])
    net.new_node_covariate('x_1')
    net.node_covariates['x_1'].data[:] = np.random.normal(2,1,net.N)
    def f_self(n_1, n_2):
        return n_1 == n_2
    net.new_edge_covariate('self_edge').from_binary_function_name(f_self)
    def f_first_half_dir(n_1, n_2):
        return (n_1 < n_2) and (n_2 in ['0','1','2','3','4'])
    net.new_edge_covariate('ec_2').from_binary_function_name(f_first_half_dir)

    print net.node_covariates['x_0']
    print net.node_covariates['x_1']
    print net.edge_covariates['self_edge']
    print net.edge_covariates['ec_2']

    print net.adjacency_matrix()
    print net.nodes()
    net.show()

    net_2 = net.subnetwork(np.array([5,0,1,6]))
    print net_2.adjacency_matrix()
    print net_2.node_covariates['x_0']
    print net_2.node_covariates['x_1']
    print net_2.edge_covariates['self_edge']
    print net_2.edge_covariates['ec_2']
    net_2.show()

    net_3 = Network(10)
    ord = np.arange(10)
    np.random.shuffle(ord)
    for i in range(10):
        for j in range(i,10):
            net_3.network[ord[i],ord[j]] = True
    net_3.offset_extremes()
    print net_3.offset.matrix()
<|endoftext|>


<commit_before>'''
This program takes a "terminal command" (should be within quotation mark) as an argument,
generate corresponding sge file, and qsub it.
'''

import argparse, subprocess

parser = argparse.ArgumentParser()
parser.add_argument("command", type=str, help="command to run")
parser.add_argument("--submit", help="submit the job", action="store_true")
args = parser.parse_args()

whether_to_qsub = args.submit
command_in_sge_file = args.command

content_for_sge_file = '''#!/bin/bash

#$ -S /bin/bash           # use bash shell
#$ -V                     # inherit the submission environment
#$ -cwd                   # start job in submission directory

#$ -m ae                 # email on abort, begin, and end
#$ -M wei.herbert.chen@gmail.com         # email address

#$ -q all.q               # queue name
#$ -l h_rt=24:00:00       # run time (hh:mm:ss)
######$ -l hostname=compute-0-3

%s

echo "This job is DONE!"

exit 0
''' % command_in_sge_file

sge_filename = '../sge_files/' + command_in_sge_file.replace(' ', '_') + '.sge'

with open(sge_filename, 'w') as sge_file:
    sge_file.write(content_for_sge_file)

if whether_to_qsub:
    subprocess.check_output(['qsub', sge_filename])
<commit_msg>fix auto_qsub, avoid problems in some symbols<commit_after>'''
This program takes a "terminal command" (should be within quotation mark) as an argument,
generate corresponding sge file, and qsub it.
'''

import argparse, subprocess

parser = argparse.ArgumentParser()
parser.add_argument("command", type=str, help="command to run")
parser.add_argument("--submit", help="submit the job", action="store_true")
args = parser.parse_args()

whether_to_qsub = args.submit
command_in_sge_file = args.command

content_for_sge_file = '''#!/bin/bash

#$ -S /bin/bash           # use bash shell
#$ -V                     # inherit the submission environment
#$ -cwd                   # start job in submission directory

#$ -m ae                 # email on abort, begin, and end
#$ -M wei.herbert.chen@gmail.com         # email address

#$ -q all.q               # queue name
#$ -l h_rt=24:00:00       # run time (hh:mm:ss)
######$ -l hostname=compute-0-3

%s

echo "This job is DONE!"

exit 0
''' % command_in_sge_file

sge_filename = '../sge_files/' + command_in_sge_file.replace(' ', '_').replace('..', '_').replace('/','_').replace('&', '') + '.sge'

with open(sge_filename, 'w') as sge_file:
    sge_file.write(content_for_sge_file)

if whether_to_qsub:
    subprocess.check_output(['qsub', sge_filename])
<|endoftext|>


<commit_before>import logging

from django.core.management import BaseCommand

from ...models import Media, Picture

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            '-f', '--force',
            default=False,
            action='store_true',
            help='Actually make changes to data (default: just print out warnings)',
        )

    def handle(self, *args, **options):
        self.check_for_empty_src(fix=options['force'])

    def check_for_empty_src(self, fix):
        for medium in Media.objects.filter(src=''):
            if medium.role == 'original':
                logger.error('original media with empty src (cannot auto-fix): %s', medium)
            else:
                picture = medium.picture
                original_media = picture.get_media('original_media')

                if original_media:
                    logger.warn('derived media with empty src: %s', medium)

                    if fix:
                        spec = medium.spec
                        medium.delete()
                        Media.create_scaled_media(original_media, spec)
                else:
                    logger.error('derived media with empty src, original missing (cannot auto-fix): %s', medium)
<commit_msg>backend/commands/fsck: make angrier<commit_after>import logging

from django.core.management import BaseCommand

from ...models import Media, Picture

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            '-f', '--force',
            default=False,
            action='store_true',
            help='Actually make changes to data (default: just print out warnings)',
        )

    def handle(self, *args, **options):
        self.check_for_empty_src(fix=options['force'])
        self.check_for_pictures_without_media(fix=options['force'])

    def check_for_empty_src(self, fix):
        media = Media.objects.filter(src='')

        if media.exists():
            if fix:
                parts = ['Media with empty src (deleting):']
            else:
                parts = ['Media with empty src (--force to delete):']

            parts.extend(str(medium) for medium in media)

            logger.warning('\n'.join(parts))

            if fix:
                media.delete()

    def check_for_pictures_without_media(self, fix):
        pictures = Picture.objects.filter(media__isnull=True)

        if pictures.exists():
            if fix:
                parts = ['Pictures without media (deleting):']
            else:
                parts = ['Non-original media with empty src (--force to delete):']

            parts.extend(str(picture) for picture in pictures)

            logger.warning('\n'.join(parts))

            if fix:
                pictures.delete()
<|endoftext|>