<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from rally import consts
from rally.plugins.openstack import scenario
from rally.plugins.openstack.scenarios.neutron import utils
from rally.task import validation

"""Scenarios for Neutron Security Groups."""

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_list_security_groups"))
class CreateAndListSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and list Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-list" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        self._create_security_group(**security_group_create_args)
        self._list_security_groups()

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_show_security_group"))
class CreateAndShowSecurityGroup(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and show Neutron security-group.

        Measure the "neutron security-group-create" and "neutron
        security-group-show" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        msg = "security_group isn't created"
        self.assertTrue(security_group, err_msg=msg)

        self._show_security_group(security_group)

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_delete_security_groups"))
class CreateAndDeleteSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and delete Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-delete" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        self._delete_security_group(security_group)

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_update_security_groups"))
class CreateAndUpdateSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None,
            security_group_update_args=None):
        """Create and update Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-update" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        :param security_group_update_args: dict, PUT /v2.0/security-groups
                                           update options
        """
        security_group_create_args = security_group_create_args or {}
        security_group_update_args = security_group_update_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        self._update_security_group(security_group,
                                    **security_group_update_args)

@validation.required_services(consts.Service.NEUTRON)
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_list_security_group_rules"))
class CreateAndListSecurityGroupRules(utils.NeutronScenario):

    def run(self, security_group_args=None,
            security_group_rule_args=None):
        """Create and list Neutron security-group-rules.

        Measure the "neutron security-group-rule-create" and "neutron
        security-group-rule-list" command performance.

        :param security_group_args: dict, POST /v2.0/security-groups
            request options
        :param security_group_rule_args: dict,
            POST /v2.0/security-group-rules request options
        """
        security_group_args = security_group_args or {}
        security_group_rule_args = security_group_rule_args or {}

        security_group = self._create_security_group(**security_group_args)
        msg = "security_group isn't created"
        self.assertTrue(security_group, err_msg=msg)

        security_group_rule = self._create_security_group_rule(
            security_group["security_group"]["id"], **security_group_rule_args)
        msg = "security_group_rule isn't created"
        self.assertTrue(security_group_rule, err_msg=msg)

        security_group_rules = self._list_security_group_rules()
        self.assertIn(security_group_rule["security_group_rule"]["id"],
                      [sgr["id"] for sgr
                       in security_group_rules["security_group_rules"]])
<commit_msg>cleanup old required_services<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from rally import consts
from rally.plugins.openstack import scenario
from rally.plugins.openstack.scenarios.neutron import utils
from rally.task import validation

"""Scenarios for Neutron Security Groups."""

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_list_security_groups"))
class CreateAndListSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and list Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-list" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        self._create_security_group(**security_group_create_args)
        self._list_security_groups()

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_show_security_group"))
class CreateAndShowSecurityGroup(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and show Neutron security-group.

        Measure the "neutron security-group-create" and "neutron
        security-group-show" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        msg = "security_group isn't created"
        self.assertTrue(security_group, err_msg=msg)

        self._show_security_group(security_group)

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_delete_security_groups"))
class CreateAndDeleteSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None):
        """Create and delete Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-delete" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        """
        security_group_create_args = security_group_create_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        self._delete_security_group(security_group)

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_update_security_groups"))
class CreateAndUpdateSecurityGroups(utils.NeutronScenario):

    def run(self, security_group_create_args=None,
            security_group_update_args=None):
        """Create and update Neutron security-groups.

        Measure the "neutron security-group-create" and "neutron
        security-group-update" command performance.

        :param security_group_create_args: dict, POST /v2.0/security-groups
                                           request options
        :param security_group_update_args: dict, PUT /v2.0/security-groups
                                           update options
        """
        security_group_create_args = security_group_create_args or {}
        security_group_update_args = security_group_update_args or {}
        security_group = self._create_security_group(
            **security_group_create_args)
        self._update_security_group(security_group,
                                    **security_group_update_args)

@validation.add("required_services",
                services=[consts.Service.NEUTRON])
@validation.add("required_platform", platform="openstack", users=True)
@scenario.configure(context={"cleanup": ["neutron"]},
                    name=("NeutronSecurityGroup"
                          ".create_and_list_security_group_rules"))
class CreateAndListSecurityGroupRules(utils.NeutronScenario):

    def run(self, security_group_args=None,
            security_group_rule_args=None):
        """Create and list Neutron security-group-rules.

        Measure the "neutron security-group-rule-create" and "neutron
        security-group-rule-list" command performance.

        :param security_group_args: dict, POST /v2.0/security-groups
            request options
        :param security_group_rule_args: dict,
            POST /v2.0/security-group-rules request options
        """
        security_group_args = security_group_args or {}
        security_group_rule_args = security_group_rule_args or {}

        security_group = self._create_security_group(**security_group_args)
        msg = "security_group isn't created"
        self.assertTrue(security_group, err_msg=msg)

        security_group_rule = self._create_security_group_rule(
            security_group["security_group"]["id"], **security_group_rule_args)
        msg = "security_group_rule isn't created"
        self.assertTrue(security_group_rule, err_msg=msg)

        security_group_rules = self._list_security_group_rules()
        self.assertIn(security_group_rule["security_group_rule"]["id"],
                      [sgr["id"] for sgr
                       in security_group_rules["security_group_rules"]])
<|endoftext|>


<commit_before>import json

from rest_framework import serializers
from rest_framework.reverse import reverse

from candidates import models as candidates_models
from images.models import Image
from elections import models as election_models
from popolo import models as popolo_models

# These are serializer classes from the Django-REST-framework API
#
# For most objects there are two serializers - a full one and a
# minimal one.  The minimal ones (whose class names begin 'Minimal')
# are used for serializing the objects when they're just being
# included as related objects, rather than the resource that
# information is being requested about.
#
# e.g. if you request information about a Post via the 'posts'
# endpoint, it's pretty useful to have the ID, URL and name of the
# elections that the Post is part of, but you probably don't need
# every bit of election metadata.  A request to the 'elections'
# endpoint, however, would include full metadata about the elections.
#
# This reduces the bloat of API responses, at the cost of some users
# having to make extra queries.

class OtherNameSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.OtherName
        fields = ('name', 'note')

class IdentifierSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Identifier
        fields = ('identifier', 'scheme')

class ContactDetailSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.ContactDetail
        fields = ('contact_type', 'label', 'note', 'value')

class LinkSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Link
        fields = ('note', 'url')

class SourceSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Source
        fields = ('note', 'url')

class AreaTypeSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = election_models.AreaType
        fields = ('id', 'url',  'name', 'source')

class AreaSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Area
        fields = (
            'id',
            'url',
            'name',
            'identifier',
            'classification',
            'other_identifiers',
            'parent',
            'type',
        )

    other_identifiers = IdentifierSerializer(many=True, read_only=True)
    type = AreaTypeSerializer(source='extra.type')

class ObjectWithImageField(serializers.RelatedField):

    def to_representation(self, value):
        kwargs = {'version': 'v0.9'}
        request = self.context['request']
        if isinstance(value, candidates_models.PersonExtra):
            kwargs.update({'pk': value.base.id})
            return reverse('person-detail', kwargs=kwargs, request=request)
        elif isinstance(value, candidates_models.OrganizationExtra):
            kwargs.update({'slug': value.slug})
            return reverse(
                'organizationextra-detail', kwargs=kwargs, request=request)
        elif isinstance(value, candidates_models.PostExtra):
            kwargs.update({'slug': value.slug})
            return reverse('postextra-detail', kwargs=kwargs, request=request)
        else:
            raise Exception("Unexpected type of object with an Image")

class ImageSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Image
        fields = (
            'id',
            'url',
            'source',
            'is_primary',
            'md5sum',
            'copyright',
            'uploading_user',
            'user_notes',
            'user_copyright',
            'notes',
            'image_url',
            'content_object',
        )

    md5sum = serializers.ReadOnlyField(source='extra.md5sum')
    copyright = serializers.ReadOnlyField(source='extra.copyright')
    uploading_user = serializers.ReadOnlyField(source='extra.uploading_user.username')
    user_notes = serializers.ReadOnlyField(source='extra.user_notes')
    user_copyright = serializers.ReadOnlyField(source='extra.user_copyright')
    notes = serializers.ReadOnlyField(source='extra.notes')
    image_url = serializers.SerializerMethodField()
    content_object = ObjectWithImageField(read_only=True)

    def get_image_url(self, i):
        return i.image.url

class MinimalOrganizationExtraSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.OrganizationExtra
        fields = ('id', 'url', 'name')

    id = serializers.ReadOnlyField(source='slug')
    name = serializers.ReadOnlyField(source='base.name')
    url = serializers.HyperlinkedIdentityField(
        view_name='organizationextra-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class PartySetSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.PartySet
        fields = ('id', 'url', 'name', 'slug')

class OrganizationExtraSerializer(MinimalOrganizationExtraSerializer):
    class Meta:
        model = candidates_models.OrganizationExtra
        fields = (
            'id',
            'url',
            'name',
            'other_names',
            'identifiers',
            'classification',
            'parent',
            'founding_date',
            'dissolution_date',
            'contact_details',
            'images',
            'links',
            'sources',
            'register',
            'party_sets',
        )

    classification = serializers.ReadOnlyField(source='base.classification')
    founding_date = serializers.ReadOnlyField(source='base.founding_date')
    dissolution_date = serializers.ReadOnlyField(source='base.dissolution_date')

    parent = MinimalOrganizationExtraSerializer(source='base.parent.extra')

    contact_details = ContactDetailSerializer(
        many=True, read_only=True, source='base.contact_details')
    identifiers = IdentifierSerializer(
        many=True, read_only=True, source='base.identifiers')
    links = LinkSerializer(
        many=True, read_only=True, source='base.links')
    other_names = OtherNameSerializer(
        many=True, read_only=True, source='base.other_names')
    sources = SourceSerializer(
        many=True, read_only=True, source='base.sources')
    images = ImageSerializer(many=True, read_only=True)

    party_sets = PartySetSerializer(
        many=True, read_only=True, source='base.party_sets')

class MinimalElectionSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = election_models.Election
        fields = ('id', 'url', 'name')

    id = serializers.ReadOnlyField(source='slug')
    url = serializers.HyperlinkedIdentityField(
        view_name='election-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class ElectionSerializer(MinimalElectionSerializer):
    class Meta:
        model = election_models.Election
        fields = (
            'id',
            'url',
            'name',
            'winner_membership_role',
            'candidate_membership_role',
            'election_date',
            'current',
            'use_for_candidate_suggestions',
            'area_types',
            'area_generation',
            'organization',
            'party_lists_in_use',
            'ocd_division',
            'description'
        )

    organization = MinimalOrganizationExtraSerializer(source='organization.extra')

    area_types = AreaTypeSerializer(many=True, read_only=True)

class MinimalPostExtraSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.PostExtra
        fields = ('id', 'url', 'label')

    id = serializers.ReadOnlyField(source='slug')
    label = serializers.ReadOnlyField(source='base.label')
    url = serializers.HyperlinkedIdentityField(
        view_name='postextra-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class MinimalPersonSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Person
        fields = ('id', 'url', 'name')

class MembershipSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Membership
        fields = (
            'id',
            'url',
            'label',
            'role',
            'person',
            'organization',
            'on_behalf_of',
            'post',
            'start_date',
            'end_date',
            'election',
        )

    person = MinimalPersonSerializer(read_only=True)
    organization = MinimalOrganizationExtraSerializer(
        read_only=True, source='organization.extra')
    on_behalf_of = MinimalOrganizationExtraSerializer(
        read_only=True, source='on_behalf_of.extra')
    post = MinimalPostExtraSerializer(
        read_only=True, source='post.extra')

    election = MinimalElectionSerializer(source='extra.election')

class JSONSerializerField(serializers.Field):
    def to_representation(self, value):
        return json.loads(value)

class PersonSerializer(MinimalPersonSerializer):
    class Meta:
        model = popolo_models.Person
        fields = (
            'id',
            'url',
            'name',
            'other_names',
            'identifiers',
            'honorific_prefix',
            'honorific_suffix',
            'sort_name',
            'email',
            'gender',
            'birth_date',
            'death_date',
            'versions',
            'contact_details',
            'links',
            'memberships',
            'images',
        )

    contact_details = ContactDetailSerializer(many=True, read_only=True)
    identifiers = IdentifierSerializer(many=True, read_only=True)
    links = LinkSerializer(many=True, read_only=True)
    other_names = OtherNameSerializer(many=True, read_only=True)
    images = ImageSerializer(many=True, read_only=True, source='extra.images')

    versions = JSONSerializerField(source='extra.versions', read_only=True)

    memberships = MembershipSerializer(many=True, read_only=True)

class PostExtraSerializer(MinimalPostExtraSerializer):
    class Meta:
        model = candidates_models.PostExtra
        fields = (
            'id',
            'url',
            'label',
            'role',
            'group',
            'candidates_locked',
            'party_set',
            'organization',
            'area',
            'elections',
            'memberships',
        )

    role = serializers.ReadOnlyField(source='base.role')
    party_set = PartySetSerializer(read_only=True)
    area = AreaSerializer(source='base.area')

    memberships = MembershipSerializer(
        many=True, read_only=True, source='base.memberships')

    organization = MinimalOrganizationExtraSerializer(
        source='base.organization.extra')

    elections = MinimalElectionSerializer(many=True, read_only=True)
<commit_msg>API: add elected and party_list_position to MembershipSerializer<commit_after>import json

from rest_framework import serializers
from rest_framework.reverse import reverse

from candidates import models as candidates_models
from images.models import Image
from elections import models as election_models
from popolo import models as popolo_models

# These are serializer classes from the Django-REST-framework API
#
# For most objects there are two serializers - a full one and a
# minimal one.  The minimal ones (whose class names begin 'Minimal')
# are used for serializing the objects when they're just being
# included as related objects, rather than the resource that
# information is being requested about.
#
# e.g. if you request information about a Post via the 'posts'
# endpoint, it's pretty useful to have the ID, URL and name of the
# elections that the Post is part of, but you probably don't need
# every bit of election metadata.  A request to the 'elections'
# endpoint, however, would include full metadata about the elections.
#
# This reduces the bloat of API responses, at the cost of some users
# having to make extra queries.

class OtherNameSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.OtherName
        fields = ('name', 'note')

class IdentifierSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Identifier
        fields = ('identifier', 'scheme')

class ContactDetailSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.ContactDetail
        fields = ('contact_type', 'label', 'note', 'value')

class LinkSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Link
        fields = ('note', 'url')

class SourceSerializer(serializers.ModelSerializer):
    class Meta:
        model = popolo_models.Source
        fields = ('note', 'url')

class AreaTypeSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = election_models.AreaType
        fields = ('id', 'url',  'name', 'source')

class AreaSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Area
        fields = (
            'id',
            'url',
            'name',
            'identifier',
            'classification',
            'other_identifiers',
            'parent',
            'type',
        )

    other_identifiers = IdentifierSerializer(many=True, read_only=True)
    type = AreaTypeSerializer(source='extra.type')

class ObjectWithImageField(serializers.RelatedField):

    def to_representation(self, value):
        kwargs = {'version': 'v0.9'}
        request = self.context['request']
        if isinstance(value, candidates_models.PersonExtra):
            kwargs.update({'pk': value.base.id})
            return reverse('person-detail', kwargs=kwargs, request=request)
        elif isinstance(value, candidates_models.OrganizationExtra):
            kwargs.update({'slug': value.slug})
            return reverse(
                'organizationextra-detail', kwargs=kwargs, request=request)
        elif isinstance(value, candidates_models.PostExtra):
            kwargs.update({'slug': value.slug})
            return reverse('postextra-detail', kwargs=kwargs, request=request)
        else:
            raise Exception("Unexpected type of object with an Image")

class ImageSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Image
        fields = (
            'id',
            'url',
            'source',
            'is_primary',
            'md5sum',
            'copyright',
            'uploading_user',
            'user_notes',
            'user_copyright',
            'notes',
            'image_url',
            'content_object',
        )

    md5sum = serializers.ReadOnlyField(source='extra.md5sum')
    copyright = serializers.ReadOnlyField(source='extra.copyright')
    uploading_user = serializers.ReadOnlyField(source='extra.uploading_user.username')
    user_notes = serializers.ReadOnlyField(source='extra.user_notes')
    user_copyright = serializers.ReadOnlyField(source='extra.user_copyright')
    notes = serializers.ReadOnlyField(source='extra.notes')
    image_url = serializers.SerializerMethodField()
    content_object = ObjectWithImageField(read_only=True)

    def get_image_url(self, i):
        return i.image.url

class MinimalOrganizationExtraSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.OrganizationExtra
        fields = ('id', 'url', 'name')

    id = serializers.ReadOnlyField(source='slug')
    name = serializers.ReadOnlyField(source='base.name')
    url = serializers.HyperlinkedIdentityField(
        view_name='organizationextra-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class PartySetSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.PartySet
        fields = ('id', 'url', 'name', 'slug')

class OrganizationExtraSerializer(MinimalOrganizationExtraSerializer):
    class Meta:
        model = candidates_models.OrganizationExtra
        fields = (
            'id',
            'url',
            'name',
            'other_names',
            'identifiers',
            'classification',
            'parent',
            'founding_date',
            'dissolution_date',
            'contact_details',
            'images',
            'links',
            'sources',
            'register',
            'party_sets',
        )

    classification = serializers.ReadOnlyField(source='base.classification')
    founding_date = serializers.ReadOnlyField(source='base.founding_date')
    dissolution_date = serializers.ReadOnlyField(source='base.dissolution_date')

    parent = MinimalOrganizationExtraSerializer(source='base.parent.extra')

    contact_details = ContactDetailSerializer(
        many=True, read_only=True, source='base.contact_details')
    identifiers = IdentifierSerializer(
        many=True, read_only=True, source='base.identifiers')
    links = LinkSerializer(
        many=True, read_only=True, source='base.links')
    other_names = OtherNameSerializer(
        many=True, read_only=True, source='base.other_names')
    sources = SourceSerializer(
        many=True, read_only=True, source='base.sources')
    images = ImageSerializer(many=True, read_only=True)

    party_sets = PartySetSerializer(
        many=True, read_only=True, source='base.party_sets')

class MinimalElectionSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = election_models.Election
        fields = ('id', 'url', 'name')

    id = serializers.ReadOnlyField(source='slug')
    url = serializers.HyperlinkedIdentityField(
        view_name='election-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class ElectionSerializer(MinimalElectionSerializer):
    class Meta:
        model = election_models.Election
        fields = (
            'id',
            'url',
            'name',
            'winner_membership_role',
            'candidate_membership_role',
            'election_date',
            'current',
            'use_for_candidate_suggestions',
            'area_types',
            'area_generation',
            'organization',
            'party_lists_in_use',
            'ocd_division',
            'description'
        )

    organization = MinimalOrganizationExtraSerializer(source='organization.extra')

    area_types = AreaTypeSerializer(many=True, read_only=True)

class MinimalPostExtraSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = candidates_models.PostExtra
        fields = ('id', 'url', 'label')

    id = serializers.ReadOnlyField(source='slug')
    label = serializers.ReadOnlyField(source='base.label')
    url = serializers.HyperlinkedIdentityField(
        view_name='postextra-detail',
        lookup_field='slug',
        lookup_url_kwarg='slug',
    )

class MinimalPersonSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Person
        fields = ('id', 'url', 'name')

class MembershipSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = popolo_models.Membership
        fields = (
            'id',
            'url',
            'label',
            'role',
            'elected',
            'party_list_position',
            'person',
            'organization',
            'on_behalf_of',
            'post',
            'start_date',
            'end_date',
            'election',
        )

    elected = serializers.ReadOnlyField(source='extra.elected')
    party_list_position = serializers.ReadOnlyField(
        source='extra.party_list_position')
    person = MinimalPersonSerializer(read_only=True)
    organization = MinimalOrganizationExtraSerializer(
        read_only=True, source='organization.extra')
    on_behalf_of = MinimalOrganizationExtraSerializer(
        read_only=True, source='on_behalf_of.extra')
    post = MinimalPostExtraSerializer(
        read_only=True, source='post.extra')

    election = MinimalElectionSerializer(source='extra.election')

class JSONSerializerField(serializers.Field):
    def to_representation(self, value):
        return json.loads(value)

class PersonSerializer(MinimalPersonSerializer):
    class Meta:
        model = popolo_models.Person
        fields = (
            'id',
            'url',
            'name',
            'other_names',
            'identifiers',
            'honorific_prefix',
            'honorific_suffix',
            'sort_name',
            'email',
            'gender',
            'birth_date',
            'death_date',
            'versions',
            'contact_details',
            'links',
            'memberships',
            'images',
        )

    contact_details = ContactDetailSerializer(many=True, read_only=True)
    identifiers = IdentifierSerializer(many=True, read_only=True)
    links = LinkSerializer(many=True, read_only=True)
    other_names = OtherNameSerializer(many=True, read_only=True)
    images = ImageSerializer(many=True, read_only=True, source='extra.images')

    versions = JSONSerializerField(source='extra.versions', read_only=True)

    memberships = MembershipSerializer(many=True, read_only=True)

class PostExtraSerializer(MinimalPostExtraSerializer):
    class Meta:
        model = candidates_models.PostExtra
        fields = (
            'id',
            'url',
            'label',
            'role',
            'group',
            'candidates_locked',
            'party_set',
            'organization',
            'area',
            'elections',
            'memberships',
        )

    role = serializers.ReadOnlyField(source='base.role')
    party_set = PartySetSerializer(read_only=True)
    area = AreaSerializer(source='base.area')

    memberships = MembershipSerializer(
        many=True, read_only=True, source='base.memberships')

    organization = MinimalOrganizationExtraSerializer(
        source='base.organization.extra')

    elections = MinimalElectionSerializer(many=True, read_only=True)
<|endoftext|>


<commit_before># Copyright (c) 2014 Novo Nordisk Foundation Center for Biosustainability, DTU.
# See LICENSE for details.

import unittest

import os

from cameo.flux_analysis.simulation import fba, pfba
from cameo.parallel import SequentialView, MultiprocessingView
from cameo.io import load_model
from cameo.flux_analysis.analysis import flux_variability_analysis, phenotypic_phase_plane, _cycle_free_fva

import pandas
from pandas.util.testing import assert_frame_equal

def assert_dataframes_equal(df, expected):
    try:
        assert_frame_equal(df, expected, check_names=False)
        return True
    except AssertionError:
        return False

TESTDIR = os.path.dirname(__file__)
REFERENCE_FVA_SOLUTION_ECOLI_CORE = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_flux_ranges_EcoliCore.csv'),
                                                    index_col=0)
REFERENCE_PPP_o2_EcoliCore = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_PPP_o2_EcoliCore.csv'))
REFERENCE_PPP_o2_glc_EcoliCore = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_PPP_o2_glc_EcoliCore.csv'))

CORE_MODEL = load_model(os.path.join(TESTDIR, 'data/EcoliCore.xml'))
iJO_MODEL = load_model(os.path.join(TESTDIR, 'data/iJO1366.xml'))

iJO_MODEL_COBRAPY = load_model(os.path.join(TESTDIR, 'data/iJO1366.xml'), solver_interface=None)

class TestFluxVariabilityAnalysis(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL
        self.biomass_flux = 0.873921
        self.model.reactions.Biomass_Ecoli_core_N_LPAREN_w_FSLASH_GAM_RPAREN__Nmet2.lower_bound = self.biomass_flux

    def test_flux_variability_sequential(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=False, view=SequentialView())
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)
        print REFERENCE_FVA_SOLUTION_ECOLI_CORE
        for key in fva_solution.index:
            self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                   REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                   REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)

    def test_flux_variability_sequential_remove_cycles(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=True, view=SequentialView())
        for key in fva_solution.index:
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)

    def test_flux_variability_parallel(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=False, view=MultiprocessingView())
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)

    @unittest.skip("multiprocessing doesn't work with cycle_free_fva yet")
    def test_flux_variability_parallel_remove_cycles(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=True, view=MultiprocessingView())
        for key in fva_solution.index:
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)

class TestPhenotypicPhasePlane(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL.copy()

    def test_one_variable(self):
        ppp = phenotypic_phase_plane(self.model, ['EX_o2_LPAREN_e_RPAREN_'])
        assert_dataframes_equal(ppp, REFERENCE_PPP_o2_EcoliCore)
        ppp = phenotypic_phase_plane(self.model, 'EX_o2_LPAREN_e_RPAREN_')
        assert_dataframes_equal(ppp, REFERENCE_PPP_o2_EcoliCore)

    def test_two_variables(self):
        ppp2d = phenotypic_phase_plane(self.model, ['EX_o2_LPAREN_e_RPAREN_', 'EX_glc_LPAREN_e_RPAREN_'],
                                       view=SequentialView())
        assert_dataframes_equal(ppp2d, REFERENCE_PPP_o2_glc_EcoliCore)

class TestSimulationMethods(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL.copy()

    def test_fba(self):
        solution = fba(self.model)
        print self.model.objective
        print self.model.solver
        self.assertAlmostEqual(solution.f, 0.873921, delta=0.000001)

    def test_pfba(self):
        fba_solution = iJO_MODEL_COBRAPY.optimize()
        print fba_solution.x_dict
        fba_flux_sum = sum((abs(val) for val in fba_solution.x_dict.values()))
        print fba_flux_sum
        solution = pfba(iJO_MODEL)
        pfba_flux_sum = sum((abs(val) for val in solution['fluxes'].values()))
        print pfba_flux_sum
        self.assertTrue(pfba_flux_sum < fba_flux_sum)

    def test_moma(self):
        pass

    def test_lmoma(self):
        pass

if __name__ == '__main__':
    import nose

    nose.runmodule()
<commit_msg>do not run PhenotypicPhasePlaneTest on travis (it kills the build)<commit_after># Copyright (c) 2014 Novo Nordisk Foundation Center for Biosustainability, DTU.
# See LICENSE for details.

import unittest

import os

from cameo.flux_analysis.simulation import fba, pfba
from cameo.parallel import SequentialView, MultiprocessingView
from cameo.io import load_model
from cameo.flux_analysis.analysis import flux_variability_analysis, phenotypic_phase_plane, _cycle_free_fva

import pandas
from pandas.util.testing import assert_frame_equal
TRAVIS = os.getenv('TRAVIS', False)

def assert_dataframes_equal(df, expected):
    try:
        assert_frame_equal(df, expected, check_names=False)
        return True
    except AssertionError:
        return False

TESTDIR = os.path.dirname(__file__)
REFERENCE_FVA_SOLUTION_ECOLI_CORE = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_flux_ranges_EcoliCore.csv'),
                                                    index_col=0)
REFERENCE_PPP_o2_EcoliCore = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_PPP_o2_EcoliCore.csv'))
REFERENCE_PPP_o2_glc_EcoliCore = pandas.read_csv(os.path.join(TESTDIR, 'data/REFERENCE_PPP_o2_glc_EcoliCore.csv'))

CORE_MODEL = load_model(os.path.join(TESTDIR, 'data/EcoliCore.xml'))
iJO_MODEL = load_model(os.path.join(TESTDIR, 'data/iJO1366.xml'))

iJO_MODEL_COBRAPY = load_model(os.path.join(TESTDIR, 'data/iJO1366.xml'), solver_interface=None)

class TestFluxVariabilityAnalysis(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL
        self.biomass_flux = 0.873921
        self.model.reactions.Biomass_Ecoli_core_N_LPAREN_w_FSLASH_GAM_RPAREN__Nmet2.lower_bound = self.biomass_flux

    def test_flux_variability_sequential(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=False, view=SequentialView())
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)
        print REFERENCE_FVA_SOLUTION_ECOLI_CORE
        for key in fva_solution.index:
            self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                   REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                   REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)

    def test_flux_variability_sequential_remove_cycles(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=True, view=SequentialView())
        for key in fva_solution.index:
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)

    def test_flux_variability_parallel(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=False, view=MultiprocessingView())
        assert_dataframes_equal(fva_solution, REFERENCE_FVA_SOLUTION_ECOLI_CORE)

    @unittest.skip("multiprocessing doesn't work with cycle_free_fva yet")
    def test_flux_variability_parallel_remove_cycles(self):
        fva_solution = flux_variability_analysis(self.model, remove_cycles=True, view=MultiprocessingView())
        for key in fva_solution.index:
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['lower_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['lower_bound'][key], delta=0.000001)
            if abs(REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key]) < 999993:
                self.assertAlmostEqual(fva_solution['upper_bound'][key],
                                       REFERENCE_FVA_SOLUTION_ECOLI_CORE['upper_bound'][key], delta=0.000001)

class TestPhenotypicPhasePlane(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL.copy()

    @unittest.skipIf(TRAVIS, 'Running in Travis')
    def test_one_variable(self):
        ppp = phenotypic_phase_plane(self.model, ['EX_o2_LPAREN_e_RPAREN_'])
        assert_dataframes_equal(ppp, REFERENCE_PPP_o2_EcoliCore)
        ppp = phenotypic_phase_plane(self.model, 'EX_o2_LPAREN_e_RPAREN_')
        assert_dataframes_equal(ppp, REFERENCE_PPP_o2_EcoliCore)

    @unittest.skipIf(TRAVIS, 'Running in Travis')
    def test_two_variables(self):
        ppp2d = phenotypic_phase_plane(self.model, ['EX_o2_LPAREN_e_RPAREN_', 'EX_glc_LPAREN_e_RPAREN_'],
                                       view=SequentialView())
        assert_dataframes_equal(ppp2d, REFERENCE_PPP_o2_glc_EcoliCore)

class TestSimulationMethods(unittest.TestCase):
    def setUp(self):
        self.model = CORE_MODEL.copy()

    def test_fba(self):
        solution = fba(self.model)
        print self.model.objective
        print self.model.solver
        self.assertAlmostEqual(solution.f, 0.873921, delta=0.000001)

    def test_pfba(self):
        fba_solution = iJO_MODEL_COBRAPY.optimize()
        print fba_solution.x_dict
        fba_flux_sum = sum((abs(val) for val in fba_solution.x_dict.values()))
        print fba_flux_sum
        solution = pfba(iJO_MODEL)
        pfba_flux_sum = sum((abs(val) for val in solution['fluxes'].values()))
        print pfba_flux_sum
        self.assertTrue(pfba_flux_sum < fba_flux_sum)

    def test_moma(self):
        pass

    def test_lmoma(self):
        pass

if __name__ == '__main__':
    import nose

    nose.runmodule()
<|endoftext|>


<commit_before># -*- coding:utf8 -*-
# File   : controller.py
# Author : Jiayuan Mao
# Email  : maojiayuan@gmail.com
# Date   : 2/28/17
#
# This file is part of TensorArtist

from . import configs, utils
from ...core.logger import get_logger
from ...core.utils.callback import CallbackManager
import zmq
import os
import collections
import threading
import queue
import contextlib
import random

logger = get_logger(__file__)

__all__ = [
    'Controller', 'control'
]

class ControllerPipeStorage(object):
    def __init__(self):
        # mapping from pipe_name => list of pipes
        self._pipes = {}

    @property
    def names(self):
        return self._pipes.keys()

    def has_pipes(self, name):
        return len(self._pipes[name]) > 0 if name in self._pipes else False

    def get_pipes(self, name):
        return self._pipes.get(name, [])

    def put_pipe(self, pipe):
        self._pipes.setdefault(pipe.name, []).append(pipe)
        return self

    @staticmethod
    def fn_filter_notempty(pipe):
        return not pipe.empty()

    def filter_notempty(self, name):
        return list(filter(ControllerPipeStorage.fn_filter_notempty, self.get_pipes(name)))

    @staticmethod
    def fn_filter_notfull(pipe):
        return not pipe.full()

    def filter_notfull(self, name):
        return list(filter(ControllerPipeStorage.fn_filter_notfull, self.get_pipes(name)))

ControlMessage = collections.namedtuple('ControlMessage', ['sock', 'identifier', 'payload', 'countdown'])
ControllerPeer = collections.namedtuple('ControllerPeer', ['info', 'csock'])
PipePeer = collections.namedtuple('PipePeer', ['addr', 'port', 'dsock'])

class Controller(object):
    def __init__(self):
        self._uid = utils.uid()
        self._addr = utils.get_addr()

        # pipes of this controller
        self._imanager = ControllerPipeStorage()
        self._omanager = ControllerPipeStorage()

        # context and poller
        self._context = zmq.Context()
        self._context.sndhwm = configs.CTL_CTL_HWM
        self._context.rcvhwm = configs.CTL_CTL_HWM
        self._poller = zmq.Poller()

        # socket pools
        self._ns_socket = None

        # control router and dispatcher
        self._control_router = None
        self._control_router_port = 0
        self._control_dispatcher = CallbackManager()
        # queue of ControlMessage
        self._control_mqueue = queue.Queue()

        # peers respect to the controller
        # map uid => ContorllerPeer
        self._controller_peers = dict()
        # the peers of input pipes (i.e. the output pipes)
        # map pipe_name => dict of <uid, PipePeer>
        self._ipipe_peers = dict()
        # the peers of output pipes
        # map pipe_name => dict of <uid, PipePeer>
        self._opipe_peers = dict()
        # map pipe_name => cache
        self._opipe_cache = dict()

        # threads and stop-event
        self._all_socks = set()
        self._all_threads = []
        self._stop_event = threading.Event()

    def socket(self, socket_type):
        sock = self._context.socket(socket_type)
        self._all_socks.add(sock)
        return sock

    def close_socket(self, sock):
        utils.graceful_close(sock)
        self._all_socks.remove(sock)
        return self

    def initialize(self, pipes=None):
        pipes = pipes or []

        for pipe in pipes:
            if pipe.direction == 'IN':
                self._imanager.put_pipe(pipe)
            else:
                assert pipe.direction == 'OUT'
                self._omanager.put_pipe(pipe)
            pipe.set_controller(self)

        # setup ns socket
        self._ns_socket = self.socket(zmq.REQ)
        self._ns_socket.connect(os.getenv(
            'TART_NAME_SERVER', '{}://localhost:{}'.format(
                configs.NS_CTL_PROTOCAL, configs.NS_CTL_PORT
            )
        ))
        self._poller.register(self._ns_socket, zmq.POLLIN)

        # setup router socket
        self._control_router = self.socket(zmq.ROUTER)
        self._control_router_port = self._control_router.bind_to_random_port('tcp://*')
        self._poller.register(self._control_router, zmq.POLLIN)

        # register on the name-server
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_REGISTER_CTL_REQ,
            'uid': self._uid,
            'ctl_protocal': 'tcp',
            'ctl_addr': self._addr,
            'ctl_port': self._control_router_port,
            'meta': {}
        })
        assert response['action'] == configs.Actions.NS_REGISTER_CTL_REP

        # register pipes on name-server
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_REGISTER_PIPE_REQ,
            'uid': self._uid,
            'ipipes': list(self._imanager.names),
            'opipes': list(self._omanager.names)
        })
        assert response['action'] == configs.Actions.NS_REGISTER_PIPE_REP

        # query name-server for ipipes
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_QUERY_OPIPE_REQ,
            'ipipes': list(self._imanager.names)
        })
        assert response['action'] == configs.Actions.NS_QUERY_OPIPE_REP
        logger.info('ipipes query {}'.format(response['results']))

        self._initialize_ipipe_peers(response['results'])

        # setup dispatcher
        self._control_dispatcher.register(configs.Actions.CTL_CONNECT_REP, self._on_ctl_connect_rep)
        self._control_dispatcher.register(configs.Actions.CTL_CONNECT_REQ, self._on_ctl_connect_req)
        self._control_dispatcher.register(configs.Actions.CTL_CONNECTED_REQ, self._on_ctl_connected_req)
        self._control_dispatcher.register(configs.Actions.CTL_NOTIFY_OPEN_REQ, self._on_ctl_notify_open_req)
        self._control_dispatcher.register(configs.Actions.CTL_NOTIFY_CLOSE_REQ, self._on_ctl_notify_close_req)

        # run threads
        self._all_threads.append(threading.Thread(target=self._main, name='ctl-main'))
        self._all_threads.append(threading.Thread(target=self._main_heartbeat, name='ctl-main-ns-heartbeat'))
        for i in self._all_threads:
            i.start()

    def finalize(self):
        self._stop_event.set()
        for i in self._all_threads:
            i.join()
        for sock in self._all_socks:
            utils.graceful_close(sock)

    def _main(self):
        wait = 0
        while True:
            socks = dict(self._poller.poll(wait))
            if self._stop_event.is_set():
                break

            nr_done = 0
            nr_done += self._main_do_control_recv(socks)
            nr_done += self._main_do_control_send()
            nr_done += self._main_do_data_recv(socks)
            nr_done += self._main_do_data_send()

            if nr_done > 0:
                wait = wait / 2 if wait > 1 else 0
            else:
                wait = wait + 1 if wait < 50 else 50

    def _main_heartbeat(self):
        while True:
            self._control_mqueue.put(ControlMessage(self._ns_socket, None, {
                'action': configs.Actions.NS_HEARTBEAT_REQ,
                'uid': self._uid
            }, countdown=0))

            if self._stop_event.wait(configs.NS_HEARTBEAT_INTERVAL):
                break

    def _main_do_control_recv(self, socks):
        nr_done = 0

        # ns
        if self._ns_socket in socks:
            for msg in utils.iter_recv(utils.req_recv_json, self._ns_socket):
                self._control_dispatcher.dispatch(msg['action'], msg)
                nr_done += 1

        # router
        if self._control_router in socks:
            for identifier, msg in utils.iter_recv(utils.router_recv_json, self._control_router):
                self._control_dispatcher.dispatch(msg['action'], identifier, msg)
                nr_done += 1

        for info, csock in self._controller_peers.values():
            if csock in socks:
                for msg in utils.iter_recv(utils.req_recv_json, csock):
                    self._control_dispatcher.dispatch(msg['action'], msg)
                    nr_done += 1

        return nr_done

    def _main_do_control_send(self):
        nr_scheduled = self._control_mqueue.qsize()
        nr_done = 0
        for i in range(nr_scheduled):
            job = self._control_mqueue.get()
            if job.identifier is not None:
                rc = utils.router_send_json(job.sock, job.identifier, job.payload, flag=zmq.NOBLOCK)
            else:
                rc = utils.req_send_json(job.sock, job.payload, flag=zmq.NOBLOCK)
            if not rc:
                if job.countdown > 0:
                    self._control_mqueue.put(ControlMessage(job[0], job[1], job[2], job.countdown - 1))
            else:
                nr_done += 1

        return nr_done

    def _main_do_data_recv(self, in_socks):
        nr_done = 0
        for name in self._imanager.names:
            pipes = self._imanager.filter_notfull(name)
            if len(pipes) == 0:
                continue

            socks= []
            for peer in self._ipipe_peers.get(name, {}).values():
                if peer.dsock in in_socks:
                    socks.append(peer.dsock)
            if len(socks) == 0:
                continue

            sock = random.choice(socks)
            msg = utils.pull_pyobj(sock)
            if msg is not None:
                for p in pipes:
                    p.put_nowait(msg['data'])
                    nr_done += 1

        return nr_done

    def _main_do_data_send(self):
        nr_done = 0
        for name in self._omanager.names:
            cache = self._opipe_cache.get(name, None)

            if cache is None:
                pipes = self._omanager.filter_notempty(name)
                if len(pipes) != 0:
                    pipe = random.choice(pipes)
                    cache = pipe.get_nowait()
                    self._opipe_cache[name] = cache

            if cache is None:
                continue

            nr_done_this = 0
            for peer in self._opipe_peers.get(name, {}).values():
                nr_done_this += utils.push_pyobj(peer.dsock, {
                    'uid': self._uid,
                    'data': cache
                }, flag=zmq.NOBLOCK)

            if nr_done_this > 0:
                self._opipe_cache[name] = None

            nr_done += nr_done_this

        return nr_done

    # BEGIN:: Connection

    def _initialize_ipipe_peers(self, results):
        for peers in results.values():
            for info in peers:
                uid = info['uid']
                if uid not in self._controller_peers:
                    self._controller_peers[uid] = ControllerPeer(info, None)
                    self._do_setup_ctl_peer(uid)

    def _do_setup_ctl_peer(self, uid):
        info, sock = self._controller_peers[uid]
        if sock is not None:
            return

        sock = self.socket(zmq.REQ)
        sock.connect('{}://{}:{}'.format(info['ctl_protocal'], info['ctl_addr'], info['ctl_port']))
        self._poller.register(sock, zmq.POLLIN)
        self._control_mqueue.put(ControlMessage(sock, None, {
            'action': configs.Actions.CTL_CONNECT_REQ,
            'uid': self._uid,
            'ipipes': list(self._imanager.names)
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        self._controller_peers[uid] = ControllerPeer(info, sock)
        logger.info('Connecting to {}'.format(uid))

    def _on_ctl_connect_req(self, identifier, msg):
        uid, pipes = msg['uid'], msg['ipipes']
        response = {}
        for name in pipes:
            if not self._omanager.has_pipes(name):
                continue

            peers = self._opipe_peers.setdefault(name, {})
            if uid in peers:
                port = peers['uid'].port
            else:
                sock = self.socket(zmq.PUSH)
                port = sock.bind_to_random_port('{}://{}'.format(configs.CTL_DAT_PROTOCAL, configs.CTL_DAT_HOST))
                peers[uid] = PipePeer(self._addr, port, sock)
                logger.info('Connection opened for {}: pipe={}, port={}'.format(uid, name, port))

            if port > 0:
                response[name] = {
                    'dat_protocal': configs.CTL_DAT_PROTOCAL,
                    'dat_addr': self._addr,
                    'dat_port': port
                }

        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_CONNECT_REP,
            'uid': self._uid,
            'opipes': response
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))

    def _on_ctl_connect_rep(self, msg):
        uid, pipes = msg['uid'], msg['opipes']

        for name, info in pipes.items():
            sock = self.socket(zmq.PULL)
            sock.connect('{}://{}:{}'.format(info['dat_protocal'], info['dat_addr'], info['dat_port']))
            self._poller.register(sock, zmq.POLLIN)
            self._ipipe_peers.setdefault(name, {})[uid] = PipePeer(info['dat_addr'], info['dat_port'], sock)
            logger.info('Connection established to {}: pipe={}, remote_port={}'.format(uid, name, info['dat_port']))

        self._control_mqueue.put(ControlMessage(self._controller_peers[uid].csock, None, {
            'action': configs.Actions.CTL_CONNECTED_REQ,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))

    def _on_ctl_connected_req(self, identifier, msg):
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_CONNECTED_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Connection established for {}'.format(msg['uid']))

    # END:: Connection

    def _on_ctl_notify_open_req(self, identifier, msg):
        uid = msg['uid']
        if uid not in self._controller_peers:
            self._controller_peers[uid] = ControllerPeer(msg['info'], None)
            self._do_setup_ctl_peer(uid)
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_NOTIFY_OPEN_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Found new controller {}'.format(uid))

    def _on_ctl_notify_close_req(self, identifier, msg):
        uid = msg['uid']
        if uid in self._controller_peers:
            peer = self._controller_peers.pop(uid)
            self._poller.unregister(peer.csock)
            self.close_socket(peer.csock)
        for peers in self._ipipe_peers.values():
            if uid in peers:
                peer = peers.pop(uid)
                self._poller.unregister(peer.dsock)
                self.close_socket(peer.dsock)
        for peers in self._opipe_peers.values():
            if uid in peers:
                peer = peers.pop(uid)
                self.close_socket(peer.dsock)
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_NOTIFY_CLOSE_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Close timeout controller {}'.format(uid))

@contextlib.contextmanager
def control(pipes):
    ctl = Controller()
    ctl.initialize(pipes)
    yield ctl
    ctl.finalize()
<commit_msg>speed up rflow.controller<commit_after># -*- coding:utf8 -*-
# File   : controller.py
# Author : Jiayuan Mao
# Email  : maojiayuan@gmail.com
# Date   : 2/28/17
#
# This file is part of TensorArtist

from . import configs, utils
from ...core.logger import get_logger
from ...core.utils.callback import CallbackManager
import zmq
import os
import collections
import threading
import queue
import contextlib
import random

logger = get_logger(__file__)

__all__ = [
    'Controller', 'control'
]

class ControllerPipeStorage(object):
    def __init__(self):
        # mapping from pipe_name => list of pipes
        self._pipes = {}

    @property
    def names(self):
        return self._pipes.keys()

    def has_pipes(self, name):
        return len(self._pipes[name]) > 0 if name in self._pipes else False

    def get_pipes(self, name):
        return self._pipes.get(name, [])

    def put_pipe(self, pipe):
        self._pipes.setdefault(pipe.name, []).append(pipe)
        return self

    @staticmethod
    def fn_filter_notempty(pipe):
        return not pipe.empty()

    def filter_notempty(self, name):
        return list(filter(ControllerPipeStorage.fn_filter_notempty, self.get_pipes(name)))

    @staticmethod
    def fn_filter_notfull(pipe):
        return not pipe.full()

    def filter_notfull(self, name):
        return list(filter(ControllerPipeStorage.fn_filter_notfull, self.get_pipes(name)))

ControlMessage = collections.namedtuple('ControlMessage', ['sock', 'identifier', 'payload', 'countdown'])
ControllerPeer = collections.namedtuple('ControllerPeer', ['info', 'csock'])
PipePeer = collections.namedtuple('PipePeer', ['addr', 'port', 'dsock'])

class Controller(object):
    def __init__(self):
        self._uid = utils.uid()
        self._addr = utils.get_addr()

        # pipes of this controller
        self._imanager = ControllerPipeStorage()
        self._omanager = ControllerPipeStorage()

        # context and poller
        self._context = zmq.Context()
        self._context.sndhwm = configs.CTL_CTL_HWM
        self._context.rcvhwm = configs.CTL_CTL_HWM
        self._poller = zmq.Poller()

        # socket pools
        self._ns_socket = None

        # control router and dispatcher
        self._control_router = None
        self._control_router_port = 0
        self._control_dispatcher = CallbackManager()
        # queue of ControlMessage
        self._control_mqueue = queue.Queue()

        # peers respect to the controller
        # map uid => ContorllerPeer
        self._controller_peers = dict()
        # the peers of input pipes (i.e. the output pipes)
        # map pipe_name => dict of <uid, PipePeer>
        self._ipipe_peers = dict()
        # the peers of output pipes
        # map pipe_name => dict of <uid, PipePeer>
        self._opipe_peers = dict()
        # map pipe_name => cache
        self._opipe_cache = dict()

        # threads and stop-event
        self._all_socks = set()
        self._all_threads = []
        self._stop_event = threading.Event()

    def socket(self, socket_type):
        sock = self._context.socket(socket_type)
        self._all_socks.add(sock)
        return sock

    def close_socket(self, sock):
        utils.graceful_close(sock)
        self._all_socks.remove(sock)
        return self

    def initialize(self, pipes=None):
        pipes = pipes or []

        for pipe in pipes:
            if pipe.direction == 'IN':
                self._imanager.put_pipe(pipe)
            else:
                assert pipe.direction == 'OUT'
                self._omanager.put_pipe(pipe)
            pipe.set_controller(self)

        # setup ns socket
        self._ns_socket = self.socket(zmq.REQ)
        self._ns_socket.connect(os.getenv(
            'TART_NAME_SERVER', '{}://localhost:{}'.format(
                configs.NS_CTL_PROTOCAL, configs.NS_CTL_PORT
            )
        ))
        self._poller.register(self._ns_socket, zmq.POLLIN)

        # setup router socket
        self._control_router = self.socket(zmq.ROUTER)
        self._control_router_port = self._control_router.bind_to_random_port('tcp://*')
        self._poller.register(self._control_router, zmq.POLLIN)

        # register on the name-server
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_REGISTER_CTL_REQ,
            'uid': self._uid,
            'ctl_protocal': 'tcp',
            'ctl_addr': self._addr,
            'ctl_port': self._control_router_port,
            'meta': {}
        })
        assert response['action'] == configs.Actions.NS_REGISTER_CTL_REP

        # register pipes on name-server
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_REGISTER_PIPE_REQ,
            'uid': self._uid,
            'ipipes': list(self._imanager.names),
            'opipes': list(self._omanager.names)
        })
        assert response['action'] == configs.Actions.NS_REGISTER_PIPE_REP

        # query name-server for ipipes
        response = utils.req_send_and_recv(self._ns_socket, {
            'action': configs.Actions.NS_QUERY_OPIPE_REQ,
            'ipipes': list(self._imanager.names)
        })
        assert response['action'] == configs.Actions.NS_QUERY_OPIPE_REP
        logger.info('ipipes query {}'.format(response['results']))

        self._initialize_ipipe_peers(response['results'])

        # setup dispatcher
        self._control_dispatcher.register(configs.Actions.CTL_CONNECT_REP, self._on_ctl_connect_rep)
        self._control_dispatcher.register(configs.Actions.CTL_CONNECT_REQ, self._on_ctl_connect_req)
        self._control_dispatcher.register(configs.Actions.CTL_CONNECTED_REQ, self._on_ctl_connected_req)
        self._control_dispatcher.register(configs.Actions.CTL_NOTIFY_OPEN_REQ, self._on_ctl_notify_open_req)
        self._control_dispatcher.register(configs.Actions.CTL_NOTIFY_CLOSE_REQ, self._on_ctl_notify_close_req)

        # run threads
        self._all_threads.append(threading.Thread(target=self._main, name='ctl-main'))
        self._all_threads.append(threading.Thread(target=self._main_heartbeat, name='ctl-main-ns-heartbeat'))
        for i in self._all_threads:
            i.start()

    def finalize(self):
        self._stop_event.set()
        for i in self._all_threads:
            i.join()
        for sock in self._all_socks:
            utils.graceful_close(sock)

    def _main(self):
        wait = 0
        while True:
            if self._stop_event.wait(wait / 1000):
                break

            socks = dict(self._poller.poll(0))

            nr_done = 0
            nr_done += self._main_do_control_recv(socks)
            nr_done += self._main_do_control_send()
            nr_done += self._main_do_data_recv(socks)
            nr_done += self._main_do_data_send()

            if nr_done > 0:
                wait = wait / 2 if wait > 1 else 0
            else:
                wait = wait + 1 if wait < 50 else 50

    def _main_heartbeat(self):
        while True:
            self._control_mqueue.put(ControlMessage(self._ns_socket, None, {
                'action': configs.Actions.NS_HEARTBEAT_REQ,
                'uid': self._uid
            }, countdown=0))

            if self._stop_event.wait(configs.NS_HEARTBEAT_INTERVAL):
                break

    def _main_do_control_recv(self, socks):
        nr_done = 0

        # ns
        if self._ns_socket in socks:
            for msg in utils.iter_recv(utils.req_recv_json, self._ns_socket):
                self._control_dispatcher.dispatch(msg['action'], msg)
                nr_done += 1

        # router
        if self._control_router in socks:
            for identifier, msg in utils.iter_recv(utils.router_recv_json, self._control_router):
                self._control_dispatcher.dispatch(msg['action'], identifier, msg)
                nr_done += 1

        for info, csock in self._controller_peers.values():
            if csock in socks:
                for msg in utils.iter_recv(utils.req_recv_json, csock):
                    self._control_dispatcher.dispatch(msg['action'], msg)
                    nr_done += 1

        return nr_done

    def _main_do_control_send(self):
        nr_scheduled = self._control_mqueue.qsize()
        nr_done = 0
        for i in range(nr_scheduled):
            job = self._control_mqueue.get()
            if job.identifier is not None:
                rc = utils.router_send_json(job.sock, job.identifier, job.payload, flag=zmq.NOBLOCK)
            else:
                rc = utils.req_send_json(job.sock, job.payload, flag=zmq.NOBLOCK)
            if not rc:
                if job.countdown > 0:
                    self._control_mqueue.put(ControlMessage(job[0], job[1], job[2], job.countdown - 1))
            else:
                nr_done += 1

        return nr_done

    def _main_do_data_recv(self, in_socks):
        nr_done = 0
        for name in self._imanager.names:
            pipes = self._imanager.filter_notfull(name)
            if len(pipes) == 0:
                continue

            socks= []
            for peer in self._ipipe_peers.get(name, {}).values():
                if peer.dsock in in_socks:
                    socks.append(peer.dsock)
            if len(socks) == 0:
                continue

            sock = random.choice(socks)
            msg = utils.pull_pyobj(sock)
            if msg is not None:
                for p in pipes:
                    p.put_nowait(msg['data'])
                    nr_done += 1

        return nr_done

    def _main_do_data_send(self):
        nr_done = 0
        for name in self._omanager.names:
            cache = self._opipe_cache.get(name, None)

            if cache is None:
                pipes = self._omanager.filter_notempty(name)
                if len(pipes) != 0:
                    pipe = random.choice(pipes)
                    cache = pipe.get_nowait()
                    self._opipe_cache[name] = cache

            if cache is None:
                continue

            nr_done_this = 0
            for peer in self._opipe_peers.get(name, {}).values():
                nr_done_this += utils.push_pyobj(peer.dsock, {
                    'uid': self._uid,
                    'data': cache
                }, flag=zmq.NOBLOCK)

            if nr_done_this > 0:
                self._opipe_cache[name] = None

            nr_done += nr_done_this

        return nr_done

    # BEGIN:: Connection

    def _initialize_ipipe_peers(self, results):
        for peers in results.values():
            for info in peers:
                uid = info['uid']
                if uid not in self._controller_peers:
                    self._controller_peers[uid] = ControllerPeer(info, None)
                    self._do_setup_ctl_peer(uid)

    def _do_setup_ctl_peer(self, uid):
        info, sock = self._controller_peers[uid]
        if sock is not None:
            return

        sock = self.socket(zmq.REQ)
        sock.connect('{}://{}:{}'.format(info['ctl_protocal'], info['ctl_addr'], info['ctl_port']))
        self._poller.register(sock, zmq.POLLIN)
        self._control_mqueue.put(ControlMessage(sock, None, {
            'action': configs.Actions.CTL_CONNECT_REQ,
            'uid': self._uid,
            'ipipes': list(self._imanager.names)
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        self._controller_peers[uid] = ControllerPeer(info, sock)
        logger.info('Connecting to {}'.format(uid))

    def _on_ctl_connect_req(self, identifier, msg):
        uid, pipes = msg['uid'], msg['ipipes']
        response = {}
        for name in pipes:
            if not self._omanager.has_pipes(name):
                continue

            peers = self._opipe_peers.setdefault(name, {})
            if uid in peers:
                port = peers['uid'].port
            else:
                sock = self.socket(zmq.PUSH)
                port = sock.bind_to_random_port('{}://{}'.format(configs.CTL_DAT_PROTOCAL, configs.CTL_DAT_HOST))
                peers[uid] = PipePeer(self._addr, port, sock)
                logger.info('Connection opened for {}: pipe={}, port={}'.format(uid, name, port))

            if port > 0:
                response[name] = {
                    'dat_protocal': configs.CTL_DAT_PROTOCAL,
                    'dat_addr': self._addr,
                    'dat_port': port
                }

        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_CONNECT_REP,
            'uid': self._uid,
            'opipes': response
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))

    def _on_ctl_connect_rep(self, msg):
        uid, pipes = msg['uid'], msg['opipes']

        for name, info in pipes.items():
            sock = self.socket(zmq.PULL)
            sock.connect('{}://{}:{}'.format(info['dat_protocal'], info['dat_addr'], info['dat_port']))
            self._poller.register(sock, zmq.POLLIN)
            self._ipipe_peers.setdefault(name, {})[uid] = PipePeer(info['dat_addr'], info['dat_port'], sock)
            logger.info('Connection established to {}: pipe={}, remote_port={}'.format(uid, name, info['dat_port']))

        self._control_mqueue.put(ControlMessage(self._controller_peers[uid].csock, None, {
            'action': configs.Actions.CTL_CONNECTED_REQ,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))

    def _on_ctl_connected_req(self, identifier, msg):
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_CONNECTED_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Connection established for {}'.format(msg['uid']))

    # END:: Connection

    def _on_ctl_notify_open_req(self, identifier, msg):
        uid = msg['uid']
        if uid not in self._controller_peers:
            self._controller_peers[uid] = ControllerPeer(msg['info'], None)
            self._do_setup_ctl_peer(uid)
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_NOTIFY_OPEN_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Found new controller {}'.format(uid))

    def _on_ctl_notify_close_req(self, identifier, msg):
        uid = msg['uid']
        if uid in self._controller_peers:
            peer = self._controller_peers.pop(uid)
            self._poller.unregister(peer.csock)
            self.close_socket(peer.csock)
        for peers in self._ipipe_peers.values():
            if uid in peers:
                peer = peers.pop(uid)
                self._poller.unregister(peer.dsock)
                self.close_socket(peer.dsock)
        for peers in self._opipe_peers.values():
            if uid in peers:
                peer = peers.pop(uid)
                self.close_socket(peer.dsock)
        self._control_mqueue.put(ControlMessage(self._control_router, identifier, {
            'action': configs.Actions.CTL_NOTIFY_CLOSE_REP,
            'uid': self._uid
        }, countdown=configs.CTL_CTL_SND_COUNTDOWN))
        logger.info('Close timeout controller {}'.format(uid))

@contextlib.contextmanager
def control(pipes):
    ctl = Controller()
    ctl.initialize(pipes)
    yield ctl
    ctl.finalize()
<|endoftext|>


<commit_before>import numpy

def ColorDeconvolution(I, W):
    '''
    Implements color deconvolution. The input image "I" consisting of RGB values is
    first transformed into optical density space, and then projected onto the stain
    vectors in the columns of "W".
    example H&E matrix
    W =array([[0.650, 0.072, 0],
              [0.704, 0.990, 0],
              [0.286, 0.105, 0]])
    *Inputs:
        I (rgbimage) - an RGB image of type unsigned char.
        W (matrix) - a 3x3 matrix containing the color vectors in columns.
                     For two stain images the third column is zero and will be
                     complemented using cross-product. Minumum two nonzero
                     columns required.
    *Outputs:
        Stains (rgbimage) - an rgb image with deconvolved stain intensities in
                            each channel, values ranging from [0, 255], suitable for display.
        StainsFloat (matrix) - an intensity image of deconvolved stains that is
                            unbounded, suitable for reconstructing color images
                            of deconvolved stains with ColorConvolution.
        Wc (matrix) - a 3x3 complemented stain matrix. Useful for color image
                      reconstruction with ColorConvolution.
    *Related functions:
        ComplementStainMatrix, OpticalDensityFwd, OpticalDensityInv, ColorConvolution
    '''

    #complement stain matrix if needed
    if(numpy.linalg.norm(W[:,2]) <= 1e-16):
        Wc = ComplementStainMatrix(W)
    else:
        Wc = W.copy()

    #normalize stains to unit-norm
    for i in range(Wc.shape[1]):
        Norm = numpy.linalg.norm(Wc[:,i])
        if(Norm >= 1e-16):
            Wc[:,i] /= Norm

    #invert stain matrix
    Q = numpy.linalg.inv(Wc)

    #transform 3D input image to 2D RGB matrix format
    m = I.shape[0]
    n = I.shape[1]
    if(I.shape[2] == 4):
        I = I[:,:,(0,1,2)]
    I = numpy.reshape(I, (m*n,3))

    #transform input RGB to optical density values and deconvolve, tfm back to RGB
    I = I.astype(dtype=numpy.float32)
    I[I == 0] = 1e-16
    ODfwd = OpticalDensityFwd(I)
    ODdeconv = numpy.dot(ODfwd, numpy.transpose(Q))
    ODinv = OpticalDensityInv(ODdeconv)

    #reshape output
    StainsFloat = numpy.reshape(ODinv, (m,n,3))

    #transform type
    Stains = numpy.copy(StainsFloat)
    Stains[Stains > 255] = 255
    Stains = Stains.astype(numpy.uint8)

    #return
    Unmixed = collections.namedtuple('Unmixed', ['Stains', 'StainsFloat', 'Wc'])
    Output = Unmixed(Stains, StainsFloat, Wc)

    return (Output)
<commit_msg>Changed docstring of ColorDeconvolution to numpy style<commit_after>import numpy

def ColorDeconvolution(I, W):
    """Performs color deconvolution.

    The given RGB Image `I` is first first transformed into optical density
    space, and then projected onto the stain vectors in the columns of the
    3x3 H&E stain matrix `W`.

    example H&E stain matrix
    W =array([[0.650, 0.072, 0],
              [0.704, 0.990, 0],
              [0.286, 0.105, 0]])

    Parameters
    -----------

    I : array_like
        An RGB Image

    W : array_like
        A 3x3 matrix containing the color vectors in columns.
        For two stain images the third column is zero and will be
        complemented using cross-product. A minumum of two nonzero
        columns required. Atleast two of the three columns must be
        non-zero.

    Returns
    --------

    Stains : array_like
        An rgb image where in each channel contains the image of the
        stain of the corresponding column in the stain matrix `W`.
        The intensity range of each channel is [0, 255] suitable for
        displaying.

    StainsFloat : array_like
        An intensity image of deconvolved stains that is unbounded,
        suitable for reconstructing color images of deconvolved stains
        with ColorConvolution.

    Wc : array_like
        A 3x3 complemented stain matrix. Useful for color image
        reconstruction with ColorConvolution.

    See Also
    --------

    ComplementStainMatrix
    OpticalDensityFwd
    OpticalDensityInv
    ColorConvolution
    """

    # complement stain matrix if needed
    if(numpy.linalg.norm(W[:, 2]) <= 1e-16):
        Wc = ComplementStainMatrix(W)
    else:
        Wc = W.copy()

    # normalize stains to unit-norm
    for i in range(Wc.shape[1]):
        Norm = numpy.linalg.norm(Wc[:, i])
        if(Norm >= 1e-16):
            Wc[:, i] /= Norm

    # invert stain matrix
    Q = numpy.linalg.inv(Wc)

    # transform 3D input image to 2D RGB matrix format
    m = I.shape[0]
    n = I.shape[1]
    if(I.shape[2] == 4):
        I = I[:, :, (0, 1, 2)]
    I = numpy.reshape(I, (m*n, 3))

    # transform input RGB to optical density values and deconvolve,
    # tfm back to RGB
    I = I.astype(dtype=numpy.float32)
    I[I == 0] = 1e-16
    ODfwd = OpticalDensityFwd(I)
    ODdeconv = numpy.dot(ODfwd, numpy.transpose(Q))
    ODinv = OpticalDensityInv(ODdeconv)

    # reshape output
    StainsFloat = numpy.reshape(ODinv, (m, n, 3))

    # transform type
    Stains = numpy.copy(StainsFloat)
    Stains[Stains > 255] = 255
    Stains = Stains.astype(numpy.uint8)

    # return
    Unmixed = collections.namedtuple('Unmixed',
                                     ['Stains', 'StainsFloat', 'Wc'])
    Output = Unmixed(Stains, StainsFloat, Wc)

    return (Output)
<|endoftext|>


<commit_before>'''
This implementation is based in John Tromp's Java implementation
of Connect 4.

https://github.com/qu1j0t3/fhourstones/blob/master/Connect4.java
'''

from __future__ import division, unicode_literals
from . import Game
from ..util import print_aec, str_aec, ZobristHashing

ROWS = 6
COLS = 7
H1 = ROWS + 1
H2 = ROWS + 2
SIZE = ROWS * COLS
SIZE1 = H1 * COLS
ALL1 = (1 << SIZE1) - 1
COL1 = (1 << H1) - 1
BOTTOM = ALL1 // COL1
TOP = BOTTOM << ROWS

class Connect4(Game):

    name = 'Connect4'
    n_positions = ROWS * COLS
    n_pieces = 2
    zobrist_hash = ZobristHashing(n_positions, n_pieces)

    def __init__(self, board=None):
        if board:
            self.board = [list(board[row*COLS:(row*COLS)+COLS]) for row in range(ROWS)]
        else:
            self.reset()

    def __hash__(self):
        return Connect4.zobrist_hash(self.board)

    def __eq__(self, other):
        attrs = ['_cur_player', '_boards', '_moves', '_height']
        return all([getattr(self, name) == getattr(other, name) for name in attrs])

    def __repr__(self):
        return Connect4View(self).render()

    def __str__(self):
        return Connect4View(self).render()

    ########
    # Game #
    ########

    def copy(self):
        c4 = Connect4()
        c4._cur_player = self._cur_player
        c4._boards = self._boards[:]
        c4._moves = self._moves[:]
        c4._height = self._height[:]
        return c4

    def cur_player(self):
        return self._cur_player

    def legal_moves(self):
        '''
        Returns the legal moves for the player in turn.
        Each move is the letter of the column where the piece
        will be placed.
        Example: ['a', 'b', 'c', 'f']
        '''
        return [chr(ord('a') + move) for move in self._moves]

    def make_move(self, move):
        '''
        A move is the letter of the column where the piece will be placed.
        '''
        move = ord(move) - ord('a')
        if move not in self._moves:
            print(type(move))
            raise Exception('Invalid move: {}'.format(move))
        new_board = self._boards[self.cur_player()] | (1 << self._height[move])
        self._height[move] += 1
        self._boards[self.cur_player()] = new_board
        self._cur_player ^= 1
        self._moves = [] if self._is_win(new_board) else self._generate_moves()
        if self._is_win(new_board):
            self._moves = []
            self._cur_player = None
        else:
            self._moves = self._generate_moves()
        return self

    def outcomes(self):
        if self._is_win(self._boards[0]):
            return ['W', 'L']
        elif self._is_win(self._boards[1]):
            return ['L', 'W']
        return ['D', 'D']

    def reset(self):
        self._cur_player = 0
        self._boards = [0, 0]
        self._height = [H1 * col for col in range(COLS)]
        self._moves = self._generate_moves()

    #############
    # Connect 4 #
    #############

    @property
    def board(self):
        b = []
        for row in range(ROWS - 1, -1, -1):
            for col in range(COLS):
                hello = (col * 7) + row
                if (1 << hello) & self._boards[0]:
                    b.append(0)
                elif (1 << hello) & self._boards[1]:
                    b.append(1)
                else:
                    b.append(' ')
        return b

    @board.setter
    def board(self, board):
        '''
        board: a string

        Example:

        game.board = (
            'O------'
            'X------'
            'O------'
            'X------'
            'O------'
            'X------'
        )
        '''
        self._boards = [0, 0]
        counters = [0, 0]
        self._height = [H1 * col for col in range(COLS)]
        max_cols =  [0] * COLS
        for row in range(ROWS):
            for col in range(COLS):
                if (board[row][col] == 'X'):
                    self._boards[0] |= 1 << ((col * COLS) + (ROWS - row - 1))
                    counters[0] += 1
                    max_cols[col] = max(max_cols[col], 6 - row)
                elif (board[row][col] == 'O'):
                    self._boards[1] |= 1 << ((col * COLS) + (ROWS - row - 1))
                    counters[1] += 1
                    max_cols[col] = max(max_cols[col], 6 - row)
        for col in range(COLS):
            self._height[col] = (col * 7) + max_cols[col]
        if self._is_win(self._boards[0]) or self._is_win(self._boards[1]):
            self._cur_player = None
            self._moves = []
            return
        diff = counters[0] - counters[1]
        if diff == 0:
            self._cur_player = 0
        elif diff == 1:
            self._cur_player = 1
        else:
            raise Exception('Illegal board!')
        self._moves = self._generate_moves()

    def _generate_moves(self):
        return [col for col in range(COLS) if ((1 << self._height[col]) & TOP) == 0]

    def _is_win(self, board):
        '''Returns Ttrue if the given bitboard has a winning pattern.'''
        y = board & (board >> ROWS)
        if y & (y >> 2 * ROWS):
            return True
        y = board & (board >> H1)
        if y & (y >> 2 * H1):
            return True
        y = board & (board >> H2)
        if y & (y >> 2 * H2):
            return True
        y = board & (board >> 1)
        if y & (y >> 2):
            return True
        return False

    #############
    # Debugging #
    #############

    def print_bitboard(self, board):
        out = '  '
        for col in range(COLS):
            out += ' ' + chr(97 + col)
        out = str_aec(out, 'bold_green') + '\n'
        for row in range(ROWS, -1, -1):
            out += ' ' + str_aec(str(row + 1), 'bold_green')
            for col in range(COLS + 10):
                hello = (col * 7) + row
                if (1 << hello) & board:
                    out += str_aec(' X', 'bold_red')
                else:
                    out += ' .'
            out += '\n'
        out += '\n'
        out += '-' * 80
        out += '\n'
        print(out)

class Connect4View(object):

    def __init__(self, game):
        self.game = game

    def _info(self):
        if self.game.is_over():
            return self._game_over() + '\n'
        return self._next_player() + self._moves() + '\n'

    def _game_over(self):
        return str_aec('Game Over!', 'bold_green') + '\n'

    def _next_player(self):
        return str_aec('Next player: ', 'bold_green') + str(self.game.cur_player()) + '\n'

    def _moves(self):
        s = '[{}]'.format(', '.join([str(s) for s in self.game.legal_moves()]))
        out = ''
        out += str_aec('Moves: ', 'bold_green') + s + '\n'
        return out

    def _board(self):
        out = '  '
        for col in range(COLS):
            out += ' ' + chr(97 + col)
        out = str_aec(out, 'bold_green') + '\n'
        for row in range(ROWS - 1, -1, -1):
            out += ' ' + str_aec(str(row + 1), 'bold_green')
            for col in range(COLS):
                hello = (col * 7) + row
                if (1 << hello) & self.game._boards[0]:
                    out += str_aec(' X', 'bold_red')
                elif (1 << hello) & self.game._boards[1]:
                    out += str_aec(' O', 'bold_blue')
                else:
                    out += ' .'
            out += '\n'
        out += '\n'
        out += '-' * 80
        out += '\n'
        return out

    def _outcomes(self):
        return '\n' + str(self.game.outcomes()) if self.game.is_over() else ''

    def render(self):
        return '{info}{board}{outcomes}'.format(
            info=self._info(),
            board=self._board(),
            outcomes=self._outcomes()
        )
<commit_msg>update<commit_after>'''
This implementation is based in John Tromp's Java implementation
of Connect 4.

https://github.com/qu1j0t3/fhourstones/blob/master/Connect4.java
'''

from __future__ import division, unicode_literals

import six
from . import Game
from ..util import print_aec, str_aec, ZobristHashing

ROWS = 6
COLS = 7
H1 = ROWS + 1
H2 = ROWS + 2
SIZE = ROWS * COLS
SIZE1 = H1 * COLS
ALL1 = (1 << SIZE1) - 1
COL1 = (1 << H1) - 1
BOTTOM = ALL1 // COL1
TOP = BOTTOM << ROWS

def board_str_to_board_mdarray(board):
    return [list(board[row*COLS:(row*COLS)+COLS]) for row in range(ROWS)]

class Connect4(Game):

    name = 'Connect4'
    n_positions = ROWS * COLS
    n_pieces = 2
    zobrist_hash = ZobristHashing(n_positions, n_pieces)

    def __init__(self, board=None):
        if board:
            if isinstance(board, six.string_types):
                self.board = board_str_to_board_mdarray(board)
        else:
            self.reset()

    def __hash__(self):
        return Connect4.zobrist_hash(self.board)

    def __eq__(self, other):
        attrs = ['_cur_player', '_boards', '_moves', '_height']
        return all([getattr(self, name) == getattr(other, name) for name in attrs])

    def __repr__(self):
        return Connect4View(self).render()

    def __str__(self):
        return Connect4View(self).render()

    ########
    # Game #
    ########

    def copy(self):
        c4 = Connect4()
        c4._cur_player = self._cur_player
        c4._boards = self._boards[:]
        c4._moves = self._moves[:]
        c4._height = self._height[:]
        return c4

    def cur_player(self):
        return self._cur_player

    def legal_moves(self):
        '''
        Returns the legal moves for the player in turn.
        Each move is the letter of the column where the piece
        will be placed.
        Example: ['a', 'b', 'c', 'f']
        '''
        return [chr(ord('a') + move) for move in self._moves]

    def make_move(self, move):
        '''
        A move is the letter of the column where the piece will be placed.
        '''
        move = ord(move) - ord('a')
        if move not in self._moves:
            print(type(move))
            raise Exception('Invalid move: {}'.format(move))
        new_board = self._boards[self.cur_player()] | (1 << self._height[move])
        self._height[move] += 1
        self._boards[self.cur_player()] = new_board
        self._cur_player ^= 1
        self._moves = [] if self._is_win(new_board) else self._generate_moves()
        if self._is_win(new_board):
            self._moves = []
            self._cur_player = None
        else:
            self._moves = self._generate_moves()
        return self

    def outcomes(self):
        if self._is_win(self._boards[0]):
            return ['W', 'L']
        elif self._is_win(self._boards[1]):
            return ['L', 'W']
        return ['D', 'D']

    def reset(self):
        self._cur_player = 0
        self._boards = [0, 0]
        self._height = [H1 * col for col in range(COLS)]
        self._moves = self._generate_moves()

    #############
    # Connect 4 #
    #############

    @property
    def board(self):
        b = []
        for row in range(ROWS - 1, -1, -1):
            for col in range(COLS):
                hello = (col * 7) + row
                if (1 << hello) & self._boards[0]:
                    b.append(0)
                elif (1 << hello) & self._boards[1]:
                    b.append(1)
                else:
                    b.append(' ')
        return b

    @board.setter
    def board(self, board):
        '''
        board: a string

        Example:

        game.board = (
            'O------'
            'X------'
            'O------'
            'X------'
            'O------'
            'X------'
        )
        '''
        self._boards = [0, 0]
        counters = [0, 0]
        self._height = [H1 * col for col in range(COLS)]
        max_cols =  [0] * COLS
        for row in range(ROWS):
            for col in range(COLS):
                if (board[row][col] == 'X'):
                    self._boards[0] |= 1 << ((col * COLS) + (ROWS - row - 1))
                    counters[0] += 1
                    max_cols[col] = max(max_cols[col], 6 - row)
                elif (board[row][col] == 'O'):
                    self._boards[1] |= 1 << ((col * COLS) + (ROWS - row - 1))
                    counters[1] += 1
                    max_cols[col] = max(max_cols[col], 6 - row)
        for col in range(COLS):
            self._height[col] = (col * 7) + max_cols[col]
        if self._is_win(self._boards[0]) or self._is_win(self._boards[1]):
            self._cur_player = None
            self._moves = []
            return
        diff = counters[0] - counters[1]
        if diff == 0:
            self._cur_player = 0
        elif diff == 1:
            self._cur_player = 1
        else:
            raise Exception('Illegal board!')
        self._moves = self._generate_moves()

    def _generate_moves(self):
        return [col for col in range(COLS) if ((1 << self._height[col]) & TOP) == 0]

    def _is_win(self, board):
        '''Returns Ttrue if the given bitboard has a winning pattern.'''
        y = board & (board >> ROWS)
        if y & (y >> 2 * ROWS):
            return True
        y = board & (board >> H1)
        if y & (y >> 2 * H1):
            return True
        y = board & (board >> H2)
        if y & (y >> 2 * H2):
            return True
        y = board & (board >> 1)
        if y & (y >> 2):
            return True
        return False

    #############
    # Debugging #
    #############

    def print_bitboard(self, board):
        out = '  '
        for col in range(COLS):
            out += ' ' + chr(97 + col)
        out = str_aec(out, 'bold_green') + '\n'
        for row in range(ROWS, -1, -1):
            out += ' ' + str_aec(str(row + 1), 'bold_green')
            for col in range(COLS + 10):
                hello = (col * 7) + row
                if (1 << hello) & board:
                    out += str_aec(' X', 'bold_red')
                else:
                    out += ' .'
            out += '\n'
        out += '\n'
        out += '-' * 80
        out += '\n'
        print(out)

class Connect4View(object):

    def __init__(self, game):
        self.game = game

    def _info(self):
        if self.game.is_over():
            return self._game_over() + '\n'
        return self._next_player() + self._moves() + '\n'

    def _game_over(self):
        return str_aec('Game Over!', 'bold_green') + '\n'

    def _next_player(self):
        return str_aec('Next player: ', 'bold_green') + str(self.game.cur_player()) + '\n'

    def _moves(self):
        s = '[{}]'.format(', '.join([str(s) for s in self.game.legal_moves()]))
        out = ''
        out += str_aec('Moves: ', 'bold_green') + s + '\n'
        return out

    def _board(self):
        out = '  '
        for col in range(COLS):
            out += ' ' + chr(97 + col)
        out = str_aec(out, 'bold_green') + '\n'
        for row in range(ROWS - 1, -1, -1):
            out += ' ' + str_aec(str(row + 1), 'bold_green')
            for col in range(COLS):
                hello = (col * 7) + row
                if (1 << hello) & self.game._boards[0]:
                    out += str_aec(' X', 'bold_red')
                elif (1 << hello) & self.game._boards[1]:
                    out += str_aec(' O', 'bold_blue')
                else:
                    out += ' .'
            out += '\n'
        out += '\n'
        out += '-' * 80
        out += '\n'
        return out

    def _outcomes(self):
        return '\n' + str(self.game.outcomes()) if self.game.is_over() else ''

    def render(self):
        return '{info}{board}{outcomes}'.format(
            info=self._info(),
            board=self._board(),
            outcomes=self._outcomes()
        )
<|endoftext|>