<commit_before>from django import forms
from django.utils import timezone

import json
import re
from datetime import timedelta

from . import models

from myvoice.survey import utils as survey_utils

VISIT_EXPR = '''
^                               # Start
\s*                             # Leading whitespace
(i|I|\d)+                       # Numbers as clinic
(\s|\*|\.)+                     # Whitespace or '*' or '.'
((1|i|I)|((i|I|o|O|[0-9])+))    # Either 1 or numbers as mobile
(\s|\*|\.)+                     # Whitespace or '*' or '.'
(i|I|o|O|[0-9])+                # Numbers as serial
(\s|\*|\.)+                     # Whitespace or '*' or '.'
(i|I|o|O|[0-9])+                # Numbers as Service
\s*                             # Trailing whitespace
$                               # End
'''
VISIT_PATT = re.compile(VISIT_EXPR, re.VERBOSE)

class VisitForm(forms.Form):
    phone = forms.CharField(max_length=20)
    text = forms.RegexField(VISIT_PATT, max_length=200, error_messages={
        'invalid': '1 or more parts of your entry are missing, please check and '
        'enter the registration again.'})

    serial_min = 3
    serial_max = 6
    min_wait_time = 1800  # Minimum time between visits by same patient in seconds

    def __init__(self, *args, **kwargs):
        super(VisitForm, self).__init__(*args, **kwargs)
        self.error_list = []

    def replace_alpha(self, text):
        """Convert 'o' and 'O' to '0', and 'i', 'I' to '1'."""
        return text.replace('o', '0').replace('O', '0').replace('i', '1').replace(
            'I', '1').replace('*', ' ').replace('.', ' ')

    def clean_text(self):
        """Validate input text.

        text is in format: CLINIC MOBILE SERIAL SERVICE
        """
        # How come this is available here?
        sender = self.cleaned_data['phone']

        cleaned_data = self.replace_alpha(self.cleaned_data['text'].strip())
        clnc, mobile, serial, srvc = cleaned_data.split()

        # Check if mobile is correct
        if len(mobile) not in [1, 11]:
            self.error_list.append('mobile')

        # Check if clinic is valid
        try:
            clinic = models.Clinic.objects.get(code=clnc)
        except (models.Clinic.DoesNotExist, ValueError):
            self.error_list.append('clinic')
            clinic = None

        # Check if serial is valid
        if len(serial) < self.serial_min or len(serial) > self.serial_max:
            self.error_list.append('serial')

        # Check if Service is valid
        try:
            service = models.Service.objects.get(code=srvc)
        except models.Service.DoesNotExist:
            self.error_list.append('service')
            service = None

        # Check if there are errors.
        # If 2 previous validation errors exist, allow entry to pass
        # As long as mobile is ok.
        if len(self.error_list) > 0:
            fld_list = ', '.join(self.error_list).upper()
            if models.VisitRegistrationError.objects.filter(
                    sender=sender).count() >= 2 and 'mobile' not in self.error_list:
                # Save error log
                models.VisitRegistrationErrorLog.objects.create(
                    sender=sender,
                    error_type=fld_list,
                    message=self.cleaned_data['text'])
                # Clear Current Error state
                models.VisitRegistrationError.objects.filter(
                    sender=sender).delete()
            else:
                # Save error state
                models.VisitRegistrationError.objects.create(sender=sender)
                error_msg = 'Error for serial {0}. There was a mistake in entering '\
                    '{1}. Please check and enter the whole registration '\
                    'code again.'.format(serial, fld_list)
                raise forms.ValidationError(error_msg)
        else:  # No errors, check if a duplicate in 30 mins
            min_wait_time = timezone.now() - timedelta(
                seconds=self.min_wait_time)
            if models.Visit.objects.filter(
                    mobile=mobile,
                    patient__serial=int(serial),  # FIXME: serial will change to text
                    patient__clinic=clinic,
                    visit_time__gt=min_wait_time).count():
                raise forms.ValidationError("Registration for patient with serial {} was"
                                            " received before. Thank you.".format(serial))

        return clinic, mobile, serial, service, self.cleaned_data['text']

class SelectClinicForm(forms.Form):
    clinic = forms.ModelChoiceField(queryset=None)

    def __init__(self, *args, **kwargs):
        super(SelectClinicForm, self).__init__(*args, **kwargs)
        self.fields['clinic'].queryset = models.Clinic.objects.order_by('name')

class FeedbackForm(forms.Form):
    """
    Requirements from TextIt Generic feedback flow
    Clinic ID response come as numeric category with label "Clinic".
    Clinic name (if clinic is not sent) comes as text with label "Which Clinic".
    Complaint message comes as text with label "General Feedback".
    Any message that comes with category "Other" is ignored.

    More:
    Clinics configured in Textit flow have corresponding code in Clinic model.
    """
    phone = forms.CharField(max_length=20)
    values = forms.CharField()

    def clean_phone(self):
        """Validate that phone is not in blocked list."""
        blocked = models.Visit.objects.exclude(sender='').values_list('sender', flat=True)
        phone = survey_utils.convert_to_local_format(self.cleaned_data['phone'])
        if phone in blocked:
            raise forms.ValidationError('The phone is not allowed')
        return self.cleaned_data['phone']

    def clean_values(self):
        """Return Clinic and Message."""
        data = self.cleaned_data['values'].replace('+', ' ')
        values = json.loads(data)

        clinic = None
        clinic_text = ''
        message = ''

        for item in values:
            category = item.get('category').lower()
            label = item.get('label').lower()
            value = item.get('value')

            if category == 'other':
                continue
            elif label == 'general feedback':
                message = value
            elif label == 'clinic':
                clinic = models.Clinic.objects.get(code=category)
            elif label == 'which clinic':
                clinic_text = value

        if clinic_text:
            message += ' ({})'.format(clinic_text)
        return {'clinic': clinic, 'message': message}
<commit_msg>removed potential bug by changing error_list from attribute of VisitForm to local variable<commit_after>from django import forms
from django.utils import timezone

import json
import re
from datetime import timedelta

from . import models

from myvoice.survey import utils as survey_utils

VISIT_EXPR = '''
^                               # Start
\s*                             # Leading whitespace
(i|I|\d)+                       # Numbers as clinic
(\s|\*|\.)+                     # Whitespace or '*' or '.'
((1|i|I)|((i|I|o|O|[0-9])+))    # Either 1 or numbers as mobile
(\s|\*|\.)+                     # Whitespace or '*' or '.'
(i|I|o|O|[0-9])+                # Numbers as serial
(\s|\*|\.)+                     # Whitespace or '*' or '.'
(i|I|o|O|[0-9])+                # Numbers as Service
\s*                             # Trailing whitespace
$                               # End
'''
VISIT_PATT = re.compile(VISIT_EXPR, re.VERBOSE)

class VisitForm(forms.Form):
    phone = forms.CharField(max_length=20)
    text = forms.RegexField(VISIT_PATT, max_length=200, error_messages={
        'invalid': '1 or more parts of your entry are missing, please check and '
        'enter the registration again.'})

    serial_min = 3
    serial_max = 6
    min_wait_time = 1800  # Minimum time between visits by same patient in seconds

    def __init__(self, *args, **kwargs):
        super(VisitForm, self).__init__(*args, **kwargs)

    def replace_alpha(self, text):
        """Convert 'o' and 'O' to '0', and 'i', 'I' to '1'."""
        return text.replace('o', '0').replace('O', '0').replace('i', '1').replace(
            'I', '1').replace('*', ' ').replace('.', ' ')

    def clean_text(self):
        """Validate input text.

        text is in format: CLINIC MOBILE SERIAL SERVICE
        """
        error_list = []
        # How come this is available here?
        sender = self.cleaned_data['phone']

        cleaned_data = self.replace_alpha(self.cleaned_data['text'].strip())
        clnc, mobile, serial, srvc = cleaned_data.split()

        # Check if mobile is correct
        if len(mobile) not in [1, 11]:
            error_list.append('mobile')

        # Check if clinic is valid
        try:
            clinic = models.Clinic.objects.get(code=clnc)
        except (models.Clinic.DoesNotExist, ValueError):
            error_list.append('clinic')
            clinic = None

        # Check if serial is valid
        if len(serial) < self.serial_min or len(serial) > self.serial_max:
            error_list.append('serial')

        # Check if Service is valid
        try:
            service = models.Service.objects.get(code=srvc)
        except models.Service.DoesNotExist:
            error_list.append('service')
            service = None

        # Check if there are errors.
        # If 2 previous validation errors exist, allow entry to pass
        # As long as mobile is ok.
        if len(error_list) > 0:
            fld_list = ', '.join(error_list).upper()
            if models.VisitRegistrationError.objects.filter(
                    sender=sender).count() >= 2 and 'mobile' not in error_list:
                # Save error log
                models.VisitRegistrationErrorLog.objects.create(
                    sender=sender,
                    error_type=fld_list,
                    message=self.cleaned_data['text'])
                # Clear Current Error state
                models.VisitRegistrationError.objects.filter(
                    sender=sender).delete()
            else:
                # Save error state
                models.VisitRegistrationError.objects.create(sender=sender)
                error_msg = 'Error for serial {0}. There was a mistake in entering '\
                    '{1}. Please check and enter the whole registration '\
                    'code again.'.format(serial, fld_list)
                raise forms.ValidationError(error_msg)
        else:  # No errors, check if a duplicate in 30 mins
            min_wait_time = timezone.now() - timedelta(
                seconds=self.min_wait_time)
            if models.Visit.objects.filter(
                    mobile=mobile,
                    patient__serial=int(serial),  # FIXME: serial will change to text
                    patient__clinic=clinic,
                    visit_time__gt=min_wait_time).count():
                raise forms.ValidationError("Registration for patient with serial {} was"
                                            " received before. Thank you.".format(serial))

        return clinic, mobile, serial, service, self.cleaned_data['text']

class SelectClinicForm(forms.Form):
    clinic = forms.ModelChoiceField(queryset=None)

    def __init__(self, *args, **kwargs):
        super(SelectClinicForm, self).__init__(*args, **kwargs)
        self.fields['clinic'].queryset = models.Clinic.objects.order_by('name')

class FeedbackForm(forms.Form):
    """
    Requirements from TextIt Generic feedback flow
    Clinic ID response come as numeric category with label "Clinic".
    Clinic name (if clinic is not sent) comes as text with label "Which Clinic".
    Complaint message comes as text with label "General Feedback".
    Any message that comes with category "Other" is ignored.

    More:
    Clinics configured in Textit flow have corresponding code in Clinic model.
    """
    phone = forms.CharField(max_length=20)
    values = forms.CharField()

    def clean_phone(self):
        """Validate that phone is not in blocked list."""
        blocked = models.Visit.objects.exclude(sender='').values_list('sender', flat=True)
        phone = survey_utils.convert_to_local_format(self.cleaned_data['phone'])
        if phone in blocked:
            raise forms.ValidationError('The phone is not allowed')
        return self.cleaned_data['phone']

    def clean_values(self):
        """Return Clinic and Message."""
        data = self.cleaned_data['values'].replace('+', ' ')
        values = json.loads(data)

        clinic = None
        clinic_text = ''
        message = ''

        for item in values:
            category = item.get('category').lower()
            label = item.get('label').lower()
            value = item.get('value')

            if category == 'other':
                continue
            elif label == 'general feedback':
                message = value
            elif label == 'clinic':
                clinic = models.Clinic.objects.get(code=category)
            elif label == 'which clinic':
                clinic_text = value

        if clinic_text:
            message += ' ({})'.format(clinic_text)
        return {'clinic': clinic, 'message': message}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

import docker

from .action import simple, script, update
from .config.client import ClientConfiguration
from .config.main import ContainerMap
from .policy.base import BasePolicy
from .runner.base import DockerClientRunner
from .state.base import SingleStateGenerator, DependencyStateGenerator, DependentStateGenerator
from .state.update import UpdateStateGenerator

log = logging.getLogger(__name__)

class MappingDockerClient(object):
    """
    Reflects a :class:`~dockermap.map.config.main.ContainerMap` instance on a Docker client
    (:class:`~docker.client.Client` or a subclass, e.g. :class:`~dockermap.map.base.DockerClientWrapper`).
    The ``policy_class`` determines if and how dependencies are being considered during creation, starting, stopping,
    and removal of containers.

    If only one container map is provided, it will be the default map for all operations. If multiple container maps
    are used, the list can contain instances of clients to be used. In that case, the ``docker_client`` argument is
    ignored.

    Image names, container status, and dependencies are cached. In order to force a refresh, use :meth:`refresh_names`.
    It is also cleared on every change of ``policy_class``.

    :param container_maps: :class:`~dockermap.map.config.main.ContainerMap` instance or a tuple or list of such
      instances along with an associated instance.
    :type container_maps: dockermap.map.config.main.ContainerMap or
      list[dockermap.map.config.main.ContainerMap] | dict[unicode | str, dockermap.map.config.main.ContainerMap]
    :param docker_client: Default :class:`~docker.client.Client` instance or configuration.
    :type docker_client: dockermap.map.config.client.ClientConfiguration or docker.client.Client
    :param clients: Dictionary of client configurations
    :type clients: dict[unicode | str, dockermap.map.config.client.ClientConfiguration]
    """
    configuration_class = ClientConfiguration
    policy_class = BasePolicy
    generators = {
        'create': (DependencyStateGenerator, simple.CreateActionGenerator),
        'start': (DependencyStateGenerator, simple.StartActionGenerator),
        'restart': (SingleStateGenerator, simple.RestartActionGenerator),
        'stop': (DependentStateGenerator, simple.StopActionGenerator),
        'remove': (DependentStateGenerator, simple.RemoveActionGenerator),
        'startup': (DependencyStateGenerator, simple.StartupActionGenerator),
        'shutdown': (DependentStateGenerator, simple.ShutdownActionGenerator),
        'update': (UpdateStateGenerator, update.UpdateActionGenerator),
        'script': (DependencyStateGenerator, script.ScriptActionGenerator),
    }
    runner_class = DockerClientRunner

    def __init__(self, container_maps=None, docker_client=None, clients=None):
        if container_maps:
            if isinstance(container_maps, ContainerMap):
                self._default_map = container_maps.name
                self._maps = {container_maps.name: container_maps}
            elif isinstance(container_maps, (list, tuple)):
                self._default_map = None
                self._maps = {c_map.name: c_map for c_map in container_maps}
            elif isinstance(container_maps, dict):
                self._default_map = None
                self._maps = container_maps
            else:
                raise ValueError("Unexpected type of 'container_maps' argument: {0}".format(type(container_maps).__name__))
        else:
            self._default_map = None
            self._maps = {}
        if clients and isinstance(clients, (list, tuple)):
            self._clients = dict(clients)
        else:
            self._clients = clients or {}

        if docker_client:
            if isinstance(docker_client, ClientConfiguration):
                default_client = docker_client
            else:
                default_client = self.configuration_class.from_client(docker_client)
            self._clients[self.policy_class.default_client_name] = default_client
        self._policy = None

    def get_policy(self):
        """
        Returns an instance of :attr:`~policy_class`.

        :return: An instance of the current policy class.
        :rtype: dockermap.map.policy.base.BasePolicy
        """
        if not self._policy:
            self._policy = self.policy_class(self._maps, self._clients)
        return self._policy

    def run_actions(self, action_name, config_name, instances=None, map_name=None, **kwargs):
        policy = self.get_policy()
        state_generator_cls, action_generator_cls = self.generators[action_name]
        state_generator = state_generator_cls(policy, kwargs)
        action_generator = action_generator_cls(policy, kwargs)
        runner = self.runner_class(policy, kwargs)
        log.debug("Passing kwargs to client actions: {0}".format(kwargs))
        results = []

        for states in state_generator.get_states(map_name or self._default_map, config_name, instances=instances):
            actions = action_generator.get_state_actions(states, **kwargs)
            log.debug("Running actions: %s", actions)
            results.extend(runner.run_actions(*actions))

        return results

    def create(self, container, instances=None, map_name=None, **kwargs):
        """
        Creates container instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance name to create. If not specified, will create all instances as specified in the
         configuration (or just one default instance).
        :type instances: tuple | list
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container creation.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('create', container, instances=instances, map_name=map_name, **kwargs)

    def start(self, container, instances=None, map_name=None, **kwargs):
        """
        Starts instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to start. If not specified, will start all instances as specified in the
         configuration (or just one default instance).
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :type instances: iterable
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container start.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('start', container, instances=instances, map_name=map_name, **kwargs)

    def restart(self, container, instances=None, map_name=None, **kwargs):
        """
        Restarts instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to stop. If not specified, will restart all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container restart.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('restart', container, instances=instances, map_name=map_name, **kwargs)

    def stop(self, container, instances=None, map_name=None, **kwargs):
        """
        Stops instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to stop. If not specified, will stop all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param raise_on_error: Errors on stop and removal may result from Docker volume problems, that do not further
          affect further actions. Such errors are always logged, but do not raise an exception unless this is set to
          ``True``. Please note that 404 errors (on non-existing containers) are always ignored on stop and removal.
        :type raise_on_error: bool
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container stop.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('stop', container, instances=instances, map_name=map_name, **kwargs)

    def remove(self, container, instances=None, map_name=None, **kwargs):
        """
        Remove instances from a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container removal.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('remove', container, instances=instances, map_name=map_name, **kwargs)

    def startup(self, container, instances=None, map_name=None, **kwargs):
        """
        Start up container instances from a container configuration. Typically this means creating and starting
        containers and their dependencies. Note that not all policy classes necessarily implement this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('startup', container, instances=instances, map_name=map_name, **kwargs)

    def shutdown(self, container, instances=None, map_name=None, **kwargs):
        """
        Shut down container instances from a container configuration. Typically this means stopping and removing
        containers. Note that not all policy classes necessarily implement this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('shutdown', container, instances=instances, map_name=map_name, **kwargs)

    def update(self, container, instances=None, map_name=None, **kwargs):
        """
        Updates instances from a container configuration. Typically this means restarting or recreating containers based
        on detected changes in the configuration or environment.  Note that not all policy classes necessarily implement
        this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('update', container, instances=instances, map_name=map_name, **kwargs)

    def call(self, action_name, container, instances=None, map_name=None, **kwargs):
        """
        Generic function for running container actions based on a policy.

        :param action_name: Action name.
        :type action_name: unicode | str
        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, runs on all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs for the policy method.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions(action_name, container, instances=instances, map_name=map_name, **kwargs)

    def run_script(self, container, instance=None, map_name=None, **kwargs):
        """
        Runs a script or single command in the context of a container. By the default implementation this means creating
        the container along with all of its dependencies, mounting the script path, and running the script. The result
        is recorded in a dictionary per client, before the container is removed. Dependencies are not removed. For
        details, see :meth:`dockermap.map.runner.script.ScriptMixin.run_script`.

        :param map_name: Container map name.
        :type map_name: unicode | str
        :param instance: Instance name. Optional, if not specified runs the default instance.
        :type instance: unicode | str
        :param container: Container configuration name.
        :type container: unicode | str
        :param kwargs: Keyword arguments to the script runner function.
        :return: A dictionary of client names with their log output and exit codes.
        :rtype: list[dict[unicode | str, unicode | str]]
        """
        return self.run_actions('script', container, instances=instance, map_name=map_name, **kwargs)

    def refresh_names(self):
        """
        Invalidates the policy name and status cache.
        """
        self._policy = None

    def list_persistent_containers(self, map_name=None):
        """
        Lists the names of all persistent containers on the specified map or all maps. Attached containers are always
        considered persistent.

        :param map_name: Container map name. Optional, only returns persistent containers from the specified map.
        :type map_name: unicode | str
        :return: List of container names.
        :rtype: list[unicode | str]
        """
        if map_name:
            maps = [self._maps[map_name].get_extended_map()]
        else:
            maps = [m.get_extended_map() for m in self._maps.values()]
        cname_func = self._policy_class.cname
        aname_func = self._policy_class.aname
        c_names = []
        for c_map in maps:
            m_name = c_map.name
            attached, persistent = c_map.get_persistent_items()
            if c_map.use_attached_parent_name:
                c_names.extend([aname_func(m_name, ca, c_name)
                                for c_name, ca in attached])
            else:
                c_names.extend([aname_func(m_name, ca[1])
                                for ca in attached])
            c_names.extend([cname_func(m_name, c_name, ci)
                            for c_name, ci in persistent])
        return c_names

    @property
    def maps(self):
        """
        Container maps.

        :return: A dictionary with container map names as keys, and the container maps values.
        :rtype: dict[unicode | str, dockermap.map.config.main.ContainerMap]
        """
        return self._maps

    @property
    def clients(self):
        """
        Clients and their configuration objects.

        :return: Dictionary of client names, with their client instance and a configuration object as values.
        :rtype: dict[unicode | str, dockermap.map.config.client.ClientConfiguration]
        """
        return self._clients

    @property
    def default_map(self):
        """
        The default map to use for any actions, if not otherwise specified.

        :return: Container map name.
        :rtype: unicode | str
        """
        return self._default_map

    @default_map.setter
    def default_map(self, value):
        if value not in self._maps:
            raise ValueError("Default name must match an available map name.")
        self._default_map = value
<commit_msg>Fixed reference to policy class.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

import docker

from .action import simple, script, update
from .config.client import ClientConfiguration
from .config.main import ContainerMap
from .policy.base import BasePolicy
from .runner.base import DockerClientRunner
from .state.base import SingleStateGenerator, DependencyStateGenerator, DependentStateGenerator
from .state.update import UpdateStateGenerator

log = logging.getLogger(__name__)

class MappingDockerClient(object):
    """
    Reflects a :class:`~dockermap.map.config.main.ContainerMap` instance on a Docker client
    (:class:`~docker.client.Client` or a subclass, e.g. :class:`~dockermap.map.base.DockerClientWrapper`).
    The ``policy_class`` determines if and how dependencies are being considered during creation, starting, stopping,
    and removal of containers.

    If only one container map is provided, it will be the default map for all operations. If multiple container maps
    are used, the list can contain instances of clients to be used. In that case, the ``docker_client`` argument is
    ignored.

    Image names, container status, and dependencies are cached. In order to force a refresh, use :meth:`refresh_names`.
    It is also cleared on every change of ``policy_class``.

    :param container_maps: :class:`~dockermap.map.config.main.ContainerMap` instance or a tuple or list of such
      instances along with an associated instance.
    :type container_maps: dockermap.map.config.main.ContainerMap or
      list[dockermap.map.config.main.ContainerMap] | dict[unicode | str, dockermap.map.config.main.ContainerMap]
    :param docker_client: Default :class:`~docker.client.Client` instance or configuration.
    :type docker_client: dockermap.map.config.client.ClientConfiguration or docker.client.Client
    :param clients: Dictionary of client configurations
    :type clients: dict[unicode | str, dockermap.map.config.client.ClientConfiguration]
    """
    configuration_class = ClientConfiguration
    policy_class = BasePolicy
    generators = {
        'create': (DependencyStateGenerator, simple.CreateActionGenerator),
        'start': (DependencyStateGenerator, simple.StartActionGenerator),
        'restart': (SingleStateGenerator, simple.RestartActionGenerator),
        'stop': (DependentStateGenerator, simple.StopActionGenerator),
        'remove': (DependentStateGenerator, simple.RemoveActionGenerator),
        'startup': (DependencyStateGenerator, simple.StartupActionGenerator),
        'shutdown': (DependentStateGenerator, simple.ShutdownActionGenerator),
        'update': (UpdateStateGenerator, update.UpdateActionGenerator),
        'script': (DependencyStateGenerator, script.ScriptActionGenerator),
    }
    runner_class = DockerClientRunner

    def __init__(self, container_maps=None, docker_client=None, clients=None):
        if container_maps:
            if isinstance(container_maps, ContainerMap):
                self._default_map = container_maps.name
                self._maps = {container_maps.name: container_maps}
            elif isinstance(container_maps, (list, tuple)):
                self._default_map = None
                self._maps = {c_map.name: c_map for c_map in container_maps}
            elif isinstance(container_maps, dict):
                self._default_map = None
                self._maps = container_maps
            else:
                raise ValueError("Unexpected type of 'container_maps' argument: {0}".format(type(container_maps).__name__))
        else:
            self._default_map = None
            self._maps = {}
        if clients and isinstance(clients, (list, tuple)):
            self._clients = dict(clients)
        else:
            self._clients = clients or {}

        if docker_client:
            if isinstance(docker_client, ClientConfiguration):
                default_client = docker_client
            else:
                default_client = self.configuration_class.from_client(docker_client)
            self._clients[self.policy_class.default_client_name] = default_client
        self._policy = None

    def get_policy(self):
        """
        Returns an instance of :attr:`~policy_class`.

        :return: An instance of the current policy class.
        :rtype: dockermap.map.policy.base.BasePolicy
        """
        if not self._policy:
            self._policy = self.policy_class(self._maps, self._clients)
        return self._policy

    def run_actions(self, action_name, config_name, instances=None, map_name=None, **kwargs):
        policy = self.get_policy()
        state_generator_cls, action_generator_cls = self.generators[action_name]
        state_generator = state_generator_cls(policy, kwargs)
        action_generator = action_generator_cls(policy, kwargs)
        runner = self.runner_class(policy, kwargs)
        log.debug("Passing kwargs to client actions: {0}".format(kwargs))
        results = []

        for states in state_generator.get_states(map_name or self._default_map, config_name, instances=instances):
            actions = action_generator.get_state_actions(states, **kwargs)
            log.debug("Running actions: %s", actions)
            results.extend(runner.run_actions(*actions))

        return results

    def create(self, container, instances=None, map_name=None, **kwargs):
        """
        Creates container instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance name to create. If not specified, will create all instances as specified in the
         configuration (or just one default instance).
        :type instances: tuple | list
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container creation.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('create', container, instances=instances, map_name=map_name, **kwargs)

    def start(self, container, instances=None, map_name=None, **kwargs):
        """
        Starts instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to start. If not specified, will start all instances as specified in the
         configuration (or just one default instance).
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :type instances: iterable
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container start.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('start', container, instances=instances, map_name=map_name, **kwargs)

    def restart(self, container, instances=None, map_name=None, **kwargs):
        """
        Restarts instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to stop. If not specified, will restart all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container restart.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('restart', container, instances=instances, map_name=map_name, **kwargs)

    def stop(self, container, instances=None, map_name=None, **kwargs):
        """
        Stops instances for a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to stop. If not specified, will stop all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param raise_on_error: Errors on stop and removal may result from Docker volume problems, that do not further
          affect further actions. Such errors are always logged, but do not raise an exception unless this is set to
          ``True``. Please note that 404 errors (on non-existing containers) are always ignored on stop and removal.
        :type raise_on_error: bool
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container stop.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('stop', container, instances=instances, map_name=map_name, **kwargs)

    def remove(self, container, instances=None, map_name=None, **kwargs):
        """
        Remove instances from a container configuration.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. If multiple actions are resulting from this, they will only be applied to
          the main container removal.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('remove', container, instances=instances, map_name=map_name, **kwargs)

    def startup(self, container, instances=None, map_name=None, **kwargs):
        """
        Start up container instances from a container configuration. Typically this means creating and starting
        containers and their dependencies. Note that not all policy classes necessarily implement this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('startup', container, instances=instances, map_name=map_name, **kwargs)

    def shutdown(self, container, instances=None, map_name=None, **kwargs):
        """
        Shut down container instances from a container configuration. Typically this means stopping and removing
        containers. Note that not all policy classes necessarily implement this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('shutdown', container, instances=instances, map_name=map_name, **kwargs)

    def update(self, container, instances=None, map_name=None, **kwargs):
        """
        Updates instances from a container configuration. Typically this means restarting or recreating containers based
        on detected changes in the configuration or environment.  Note that not all policy classes necessarily implement
        this method.

        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, will remove all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs. Only options controlling policy behavior are considered.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions('update', container, instances=instances, map_name=map_name, **kwargs)

    def call(self, action_name, container, instances=None, map_name=None, **kwargs):
        """
        Generic function for running container actions based on a policy.

        :param action_name: Action name.
        :type action_name: unicode | str
        :param container: Container name.
        :type container: unicode | str
        :param instances: Instance names to remove. If not specified, runs on all instances as specified in the
         configuration (or just one default instance).
        :type instances: iterable
        :param map_name: Container map name. Optional - if not provided the default map is used.
        :type map_name: unicode | str
        :param kwargs: Additional kwargs for the policy method.
        :return: Return values of created main containers.
        :rtype: list[(unicode | str, dict)]
        """
        return self.run_actions(action_name, container, instances=instances, map_name=map_name, **kwargs)

    def run_script(self, container, instance=None, map_name=None, **kwargs):
        """
        Runs a script or single command in the context of a container. By the default implementation this means creating
        the container along with all of its dependencies, mounting the script path, and running the script. The result
        is recorded in a dictionary per client, before the container is removed. Dependencies are not removed. For
        details, see :meth:`dockermap.map.runner.script.ScriptMixin.run_script`.

        :param map_name: Container map name.
        :type map_name: unicode | str
        :param instance: Instance name. Optional, if not specified runs the default instance.
        :type instance: unicode | str
        :param container: Container configuration name.
        :type container: unicode | str
        :param kwargs: Keyword arguments to the script runner function.
        :return: A dictionary of client names with their log output and exit codes.
        :rtype: list[dict[unicode | str, unicode | str]]
        """
        return self.run_actions('script', container, instances=instance, map_name=map_name, **kwargs)

    def refresh_names(self):
        """
        Invalidates the policy name and status cache.
        """
        self._policy = None

    def list_persistent_containers(self, map_name=None):
        """
        Lists the names of all persistent containers on the specified map or all maps. Attached containers are always
        considered persistent.

        :param map_name: Container map name. Optional, only returns persistent containers from the specified map.
        :type map_name: unicode | str
        :return: List of container names.
        :rtype: list[unicode | str]
        """
        if map_name:
            maps = [self._maps[map_name].get_extended_map()]
        else:
            maps = [m.get_extended_map() for m in self._maps.values()]
        cname_func = self.policy_class.cname
        aname_func = self.policy_class.aname
        c_names = []
        for c_map in maps:
            m_name = c_map.name
            attached, persistent = c_map.get_persistent_items()
            if c_map.use_attached_parent_name:
                c_names.extend([aname_func(m_name, ca, c_name)
                                for c_name, ca in attached])
            else:
                c_names.extend([aname_func(m_name, ca[1])
                                for ca in attached])
            c_names.extend([cname_func(m_name, c_name, ci)
                            for c_name, ci in persistent])
        return c_names

    @property
    def maps(self):
        """
        Container maps.

        :return: A dictionary with container map names as keys, and the container maps values.
        :rtype: dict[unicode | str, dockermap.map.config.main.ContainerMap]
        """
        return self._maps

    @property
    def clients(self):
        """
        Clients and their configuration objects.

        :return: Dictionary of client names, with their client instance and a configuration object as values.
        :rtype: dict[unicode | str, dockermap.map.config.client.ClientConfiguration]
        """
        return self._clients

    @property
    def default_map(self):
        """
        The default map to use for any actions, if not otherwise specified.

        :return: Container map name.
        :rtype: unicode | str
        """
        return self._default_map

    @default_map.setter
    def default_map(self, value):
        if value not in self._maps:
            raise ValueError("Default name must match an available map name.")
        self._default_map = value
<|endoftext|>


<commit_before>"""
Testing Recursive feature elimination
"""

import numpy as np
from numpy.testing import assert_array_almost_equal

from ..rfe import RFE, RFECV
from ...datasets import load_iris
from ...metrics import zero_one
from ...svm import SVC
from ...utils import check_random_state

def test_rfe():
    generator = check_random_state(0)

    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    rfe = RFE(estimator=SVC(kernel="linear"), n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])

def test_rfecv():
    generator = check_random_state(0)

    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    # Test using the score function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=3)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])

    # Test using a customized loss function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=3, loss_func=zero_one)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])
<commit_msg>Improved test coverage to 100%<commit_after>"""
Testing Recursive feature elimination
"""

import numpy as np
from numpy.testing import assert_array_almost_equal

from ..rfe import RFE, RFECV
from ...datasets import load_iris
from ...metrics import zero_one
from ...svm import SVC
from ...utils import check_random_state

def test_rfe():
    generator = check_random_state(0)

    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    clf = SVC(kernel="linear")
    rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
    rfe.fit(X, y)
    X_r = rfe.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])

    assert_array_almost_equal(rfe.predict(X), clf.predict(iris.data))
    assert rfe.score(X, y) == clf.score(iris.data, iris.target)

def test_rfecv():
    generator = check_random_state(0)

    iris = load_iris()
    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
    y = iris.target

    # Test using the score function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=3)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])

    # Test using a customized loss function
    rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, cv=3, loss_func=zero_one)
    rfecv.fit(X, y)
    X_r = rfecv.transform(X)

    assert X_r.shape == iris.data.shape
    assert_array_almost_equal(X_r[:10], iris.data[:10])
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import os
import sys
import shutil
import numpy as np
from tqdm import tqdm

import torch
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
from sklearn.metrics import accuracy_score, mean_squared_error

from model import Classifier
from dataset import VisualCommunicationDataset

def save_checkpoint(state, is_best, folder='./', filename='checkpoint.pth.tar'):
    if not os.path.isdir(folder):
        os.mkdir(folder)
    torch.save(state, os.path.join(folder, filename))
    if is_best:
        shutil.copyfile(os.path.join(folder, filename),
                        os.path.join(folder, 'model_best.pth.tar'))

def load_checkpoint(file_path, use_cuda=False):
    checkpoint = torch.load(file_path) if use_cuda else \
        torch.load(file_path, map_location=lambda storage, location: storage)
    model = Classifier(checkpoint['distance'])
    model.load_state_dict(checkpoint['state_dict'])
    return model

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--soft-labels', action='store_true', default=False,
                        help='use soft or hard labels [default: False]')
    parser.add_argument('--out-dir', type=str, default='./trained_models',
                        help='where to save checkpoints [./trained_models]')
    parser.add_argument('--distance', type=str, default='cosine', help='cosine|euclidean')
    parser.add_argument('--batch-size', type=int, default=10,
                        help='number of examples in a mini-batch [default: 10]')
    parser.add_argument('--lr', type=float, default=1e-3, help='learning rate [default: 1e-3]')
    parser.add_argument('--epochs', type=int, default=10, help='number of epochs [default: 10]')
    parser.add_argument('--cuda', action='store_true', default=False)
    args = parser.parse_args()
    args.cuda = args.cuda and torch.cuda.is_available()

    train_dataset = VisualCommunicationDataset(layer='conv42', split='train', soft_labels=args.soft_labels)
    val_dataset = VisualCommunicationDataset(layer='conv42', split='val', soft_labels=args.soft_labels)
    test_dataset = VisualCommunicationDataset(layer='conv42', split='test', soft_labels=args.soft_labels)
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False)

    model = Classifier(args.distance)
    if args.cuda:
        model.cuda()
    optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)

    def train(epoch):
        model.train()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()

        for batch_idx, (photo, sketch, label) in enumerate(train_loader):
            photo = Variable(photo)
            sketch = Variable(sketch)
            label = Variable(label)
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            optimizer.zero_grad()
            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)

            loss.backward()
            optimizer.step()

            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\t{}: {:2f}'.format(
                epoch, batch_idx * batch_size, len(train_loader.dataset),  100. * batch_idx / len(train_loader),
                loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy' , metric_meter.avg))

        print('====> Epoch: {}\tLoss: {:.4f}\tAcc: {:.2f}'.format(epoch, loss_meter.avg, metric_meter.avg))
	return loss_meter.avg, metric_meter.avg

    def validate():
        model.eval()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()
        pbar = tqdm(total=len(val_loader))

        for batch_idx, (photo, sketch, label) in enumerate(val_loader):
            photo = Variable(photo, volatile=True)
            sketch = Variable(sketch, volatile=True)
            label = Variable(label, requires_grad=False).float()
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)
            pbar.update()

        pbar.close()
        print('====> Val Loss: {:.4f}\tVal {}: {:.2f}'.format(
            loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy', metric_meter.avg))
        return loss_meter.avg, metric_meter.avg

    def test():
        model.eval()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()
        pbar = tqdm(total=len(test_loader))

        for batch_idx, (photo, sketch, label) in enumerate(test_loader):
            photo = Variable(photo, volatile=True)
            sketch = Variable(sketch, volatile=True)
            label = Variable(label, requires_grad=False).float()
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)
            pbar.update()

        pbar.close()
        print('====> Test Loss: {:.4f}\tTest {}: {:.2f}'.format(
            loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy', metric_meter.avg))
        return loss_meter.avg, metric_meter.avg

    loss_db = np.zeros((args.epochs, 3))
    metric_db = np.zeros((args.epochs, 3))
    best_loss = sys.maxint
    for epoch in xrange(1, args.epochs + 1):
        train_loss, train_metric = train(epoch)
        val_loss, val_metric = validate()
        test_loss, test_metric = test()
        is_best = val_loss < best_loss
        best_loss = min(val_loss, best_loss)
        save_checkpoint({
            'state_dict': model.state_dict(),
            'distance': args.distance,
            'train_loss': train_loss,
            'train_metric': train_metric,
            'val_loss': val_loss,
            'val_metric': val_metric,
            'test_loss': test_loss,
            'test_metric': test_metric,
            'optimizer' : optimizer.state_dict(),
        }, is_best, folder=args.out_dir)
        # save stuff for plots
        loss_db[epoch - 1, 0] = train_loss
        loss_db[epoch - 1, 1] = val_loss
        loss_db[epoch - 1, 2] = test_loss
        metric_db[epoch - 1, 0] = train_metric
        metric_db[epoch - 1, 1] = val_metric
        metric_db[epoch - 1, 2] = test_metric

    # plot the training numbers
    import matplotlib.pyplot as plt
    plt.switch_backend('Agg')
    import seaborn as sns
    sns.set_style('whitegrid')
    plt.figure()
    plt.plot(loss_db[:, 0], '-', label='Train')
    plt.plot(loss_db[:, 1], '-', label='Val')
    plt.plot(loss_db[:, 2], '-', label='Test')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.tight_layout()
    plt.savefig(os.path.join(args.out_dir, 'loss.png'))

    plt.figure()
    plt.plot(metric_db[:, 0], '-', label='Train')
    plt.plot(metric_db[:, 1], '-', label='Val')
    plt.plot(metric_db[:, 2], '-', label='Test')
    plt.xlabel('Epoch')
    plt.ylabel('MSE' if args.soft_labels else 'Accuracy')
    plt.legend()
    plt.tight_layout()
    plt.savefig(os.path.join(args.out_dir, 'metric.png'))
<commit_msg>soft-labels seem like they are still too hard to fit<commit_after>from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import os
import sys
import shutil
import numpy as np
from tqdm import tqdm

import torch
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
from sklearn.metrics import accuracy_score, mean_squared_error

from model import Classifier
from dataset import VisualCommunicationDataset

def save_checkpoint(state, is_best, folder='./', filename='checkpoint.pth.tar'):
    if not os.path.isdir(folder):
        os.mkdir(folder)
    torch.save(state, os.path.join(folder, filename))
    if is_best:
        shutil.copyfile(os.path.join(folder, filename),
                        os.path.join(folder, 'model_best.pth.tar'))

def load_checkpoint(file_path, use_cuda=False):
    checkpoint = torch.load(file_path) if use_cuda else \
        torch.load(file_path, map_location=lambda storage, location: storage)
    model = Classifier(checkpoint['distance'])
    model.load_state_dict(checkpoint['state_dict'])
    return model

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--soft-labels', action='store_true', default=False,
                        help='use soft or hard labels [default: False]')
    parser.add_argument('--out-dir', type=str, default='./trained_models',
                        help='where to save checkpoints [./trained_models]')
    parser.add_argument('--distance', type=str, default='cosine', help='cosine|euclidean')
    parser.add_argument('--batch-size', type=int, default=10,
                        help='number of examples in a mini-batch [default: 10]')
    parser.add_argument('--lr', type=float, default=1e-3, help='learning rate [default: 1e-3]')
    parser.add_argument('--epochs', type=int, default=10, help='number of epochs [default: 10]')
    parser.add_argument('--cuda', action='store_true', default=False)
    args = parser.parse_args()
    args.cuda = args.cuda and torch.cuda.is_available()

    train_dataset = VisualCommunicationDataset(layer='conv42', split='train', soft_labels=args.soft_labels)
    val_dataset = VisualCommunicationDataset(layer='conv42', split='val', soft_labels=args.soft_labels)
    test_dataset = VisualCommunicationDataset(layer='conv42', split='test', soft_labels=args.soft_labels)
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False)

    model = Classifier(args.distance)
    if args.cuda:
        model.cuda()
    optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)

    def train(epoch):
        model.train()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()

        for batch_idx, (photo, sketch, label) in enumerate(train_loader):
            photo = Variable(photo)
            sketch = Variable(sketch)
            label = Variable(label)
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            optimizer.zero_grad()
            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)

            loss.backward()
            optimizer.step()

            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\t{}: {:6f}'.format(
                epoch, batch_idx * batch_size, len(train_loader.dataset),  100. * batch_idx / len(train_loader),
                loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy' , metric_meter.avg))

        print('====> Epoch: {}\tLoss: {:.4f}\t{}: {:.6f}'.format(
            epoch, loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy', metric_meter.avg))
	return loss_meter.avg, metric_meter.avg

    def validate():
        model.eval()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()
        pbar = tqdm(total=len(val_loader))

        for batch_idx, (photo, sketch, label) in enumerate(val_loader):
            photo = Variable(photo, volatile=True)
            sketch = Variable(sketch, volatile=True)
            label = Variable(label, requires_grad=False).float()
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)
            pbar.update()

        pbar.close()
        print('====> Val Loss: {:.4f}\tVal {}: {:.6f}'.format(
            loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy', metric_meter.avg))
        return loss_meter.avg, metric_meter.avg

    def test():
        model.eval()
        loss_meter = AverageMeter()
        metric_meter = AverageMeter()
        pbar = tqdm(total=len(test_loader))

        for batch_idx, (photo, sketch, label) in enumerate(test_loader):
            photo = Variable(photo, volatile=True)
            sketch = Variable(sketch, volatile=True)
            label = Variable(label, requires_grad=False).float()
            batch_size = len(photo)

            if args.cuda:
                photo = photo.cuda()
                sketch = sketch.cuda()
                label = label.cuda()

            photo = photo.view(batch_size * 4, 512, 28, 28)
            sketch = sketch.view(batch_size * 4, 512, 28, 28)
            label = label.view(batch_size * 4, 1)

            pred = model(photo, sketch)
            loss = F.binary_cross_entropy(pred, label)
            loss_meter.update(loss.data[0], batch_size)

            if args.soft_labels:
                label_np = label.cpu().data.numpy()
                pred_np = pred.cpu().data.numpy()
                mse = mean_squared_error(label_np, pred_np)
                metric_meter.update(mse, batch_size)
            else:
                label_np = np.round(label.cpu().data.numpy(), 0)
                pred_np = np.round(pred.cpu().data.numpy(), 0)
                acc = accuracy_score(label_np, pred_np)
                metric_meter.update(acc, batch_size)
            pbar.update()

        pbar.close()
        print('====> Test Loss: {:.4f}\tTest {}: {:.6f}'.format(
            loss_meter.avg, 'MSE' if args.soft_labels else 'Accuracy', metric_meter.avg))
        return loss_meter.avg, metric_meter.avg

    loss_db = np.zeros((args.epochs, 3))
    metric_db = np.zeros((args.epochs, 3))
    best_loss = sys.maxint
    for epoch in xrange(1, args.epochs + 1):
        train_loss, train_metric = train(epoch)
        val_loss, val_metric = validate()
        test_loss, test_metric = test()
        is_best = val_loss < best_loss
        best_loss = min(val_loss, best_loss)
        save_checkpoint({
            'state_dict': model.state_dict(),
            'distance': args.distance,
            'train_loss': train_loss,
            'train_metric': train_metric,
            'val_loss': val_loss,
            'val_metric': val_metric,
            'test_loss': test_loss,
            'test_metric': test_metric,
            'optimizer' : optimizer.state_dict(),
        }, is_best, folder=args.out_dir)
        # save stuff for plots
        loss_db[epoch - 1, 0] = train_loss
        loss_db[epoch - 1, 1] = val_loss
        loss_db[epoch - 1, 2] = test_loss
        metric_db[epoch - 1, 0] = train_metric
        metric_db[epoch - 1, 1] = val_metric
        metric_db[epoch - 1, 2] = test_metric

    # plot the training numbers
    import matplotlib.pyplot as plt
    plt.switch_backend('Agg')
    import seaborn as sns
    sns.set_style('whitegrid')
    plt.figure()
    plt.plot(loss_db[:, 0], '-', label='Train')
    plt.plot(loss_db[:, 1], '-', label='Val')
    plt.plot(loss_db[:, 2], '-', label='Test')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.tight_layout()
    plt.savefig(os.path.join(args.out_dir, 'loss.png'))

    plt.figure()
    plt.plot(metric_db[:, 0], '-', label='Train')
    plt.plot(metric_db[:, 1], '-', label='Val')
    plt.plot(metric_db[:, 2], '-', label='Test')
    plt.xlabel('Epoch')
    plt.ylabel('MSE' if args.soft_labels else 'Accuracy')
    plt.legend()
    plt.tight_layout()
    plt.savefig(os.path.join(args.out_dir, 'metric.png'))
<|endoftext|>