<commit_before>"""Constructs deep network reward models."""

import abc
from typing import Callable, Iterable, Sequence

import gym
import numpy as np
import torch as th
from stable_baselines3.common import preprocessing
from torch import nn

import imitation.rewards.common as rewards_common
from imitation.util import networks

class RewardNet(nn.Module, abc.ABC):
    """Minimal abstract reward network.

    Only requires the implementation of a forward pass (calculating rewards given
    a batch of states, actions, next states and dones).
    """

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        normalize_images: bool = True,
    ):
        """Initialize the RewardNet.

        Args:
            observation_space: the observation space of the environment
            action_space: the action space of the environment
            normalize_images: whether to automatically normalize
                image observations to [0, 1] (from 0 to 255). Defaults to True.
        """
        super().__init__()
        self.observation_space = observation_space
        self.action_space = action_space
        self.normalize_images = normalize_images

    @abc.abstractmethod
    def forward(
        self,
        state: th.Tensor,
        action: th.Tensor,
        next_state: th.Tensor,
        done: th.Tensor,
    ):
        """Compute rewards for a batch of transitions and keep gradients."""

    def predict(
        self,
        state: np.ndarray,
        action: np.ndarray,
        next_state: np.ndarray,
        done: np.ndarray,
    ) -> np.ndarray:
        """Compute rewards for a batch of transitions without gradients.
        Also performs some preprocessing and numpy conversion.
        """
        (
            state_th,
            action_th,
            next_state_th,
            done_th,
        ) = rewards_common.disc_rew_preprocess_inputs(
            observation_space=self.observation_space,
            action_space=self.action_space,
            state=state,
            action=action,
            next_state=next_state,
            done=done,
            device=self.device,
            normalize_images=self.normalize_images,
        )

        with th.no_grad():
            rew_th = self(state_th, action_th, next_state_th, done_th)

        rew = rew_th.detach().cpu().numpy().flatten()
        assert rew.shape == state.shape[:1]
        return rew

    @property
    def device(self) -> th.device:
        """Heuristic to determine which device this module is on."""
        try:
            first_param = next(self.parameters())
            return first_param.device
        except StopIteration:
            # if the model has no parameters, we use the CPU
            return th.device("cpu")

class ShapedRewardNet(RewardNet):
    """A RewardNet consisting of a base net and a potential shaping."""

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        base: RewardNet,
        potential: Callable[[th.Tensor], th.Tensor],
        discount_factor: float,
        normalize_images: bool = True,
    ):
        """Setup a ShapedRewardNet instance.

        Args:
            observation_space: observation space of the environment
            action_space: action space of the environment
            base: the base reward net to which the potential shaping
                will be added.
            potential: A callable which takes
                a batch of states (as a PyTorch tensor) and returns a batch of
                potentials for these states. If this is a PyTorch Module, it becomes
                a submodule of the ShapedRewardNet instance.
            discount_factor: discount factor to use for the potential shaping
            normalize_images: passed through to `RewardNet.__init__`,
                see its documentation
        """
        super().__init__(
            observation_space=observation_space,
            action_space=action_space,
            normalize_images=normalize_images,
        )
        self.base = base
        self.potential = potential
        self.discount_factor = discount_factor

        # end_potential is the potential when the episode terminates.
        if discount_factor == 1.0:
            # If undiscounted, terminal state must have potential 0.
            self.end_potential = 0.0
        else:
            # Otherwise, it can be arbitrary, so make a trainable variable.
            self.end_potential = nn.Parameter(th.zeros(()))

    def forward(
        self,
        state: th.Tensor,
        action: th.Tensor,
        next_state: th.Tensor,
        done: th.Tensor,
    ):
        base_reward_net_output = self.base(state, action, next_state, done)
        new_shaping_output = self.potential(next_state).flatten()
        old_shaping_output = self.potential(state).flatten()
        done_f = done.float()
        new_shaping = done_f * self.end_potential + (1 - done_f) * new_shaping_output
        final_rew = (
            base_reward_net_output
            + self.discount_factor * new_shaping
            - old_shaping_output
        )
        assert final_rew.shape == state.shape[:1]
        return final_rew

class BasicRewardNet(RewardNet):
    """MLP that flattens and concatenates current state, current action, next state, and
    done flag, depending on given `use_*` keyword arguments."""

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        use_state: bool = True,
        use_action: bool = True,
        use_next_state: bool = False,
        use_done: bool = False,
        **kwargs,
    ):
        """Builds reward MLP.

        Args:
          observation_space: The observation space.
          action_space: The action space.
          use_state: should the current state be included as an input to the MLP?
          use_action: should the current action be included as an input to the MLP?
          use_next_state: should the next state be included as an input to the MLP?
          use_done: should the "done" flag be included as an input to the MLP?
          kwargs: passed straight through to build_mlp.
        """
        super().__init__(observation_space, action_space)
        combined_size = 0

        self.use_state = use_state
        if self.use_state:
            combined_size += preprocessing.get_flattened_obs_dim(observation_space)

        self.use_action = use_action
        if self.use_action:
            combined_size += preprocessing.get_flattened_obs_dim(action_space)

        self.use_next_state = use_next_state
        if self.use_next_state:
            combined_size += preprocessing.get_flattened_obs_dim(observation_space)

        self.use_done = use_done
        if self.use_done:
            combined_size += 1

        full_build_mlp_kwargs = {
            "hid_sizes": (32, 32),
        }
        full_build_mlp_kwargs.update(kwargs)
        full_build_mlp_kwargs.update(
            {
                # we do not want these overridden
                "in_size": combined_size,
                "out_size": 1,
                "squeeze_output": True,
            }
        )

        self.mlp = networks.build_mlp(**full_build_mlp_kwargs)

    def forward(self, state, action, next_state, done):
        inputs = []
        if self.use_state:
            inputs.append(th.flatten(state, 1))
        if self.use_action:
            inputs.append(th.flatten(action, 1))
        if self.use_next_state:
            inputs.append(th.flatten(next_state, 1))
        if self.use_done:
            inputs.append(th.reshape(done, [-1, 1]))

        inputs_concat = th.cat(inputs, dim=1)

        outputs = self.mlp(inputs_concat)
        assert outputs.shape == state.shape[:1]

        return outputs

class BasicShapedRewardNet(ShapedRewardNet):
    """Shaped reward net based on MLPs.

    This is just a very simple convenience class for instantiating a BasicRewardNet
    and a BasicPotentialShaping and wrapping them inside a ShapedRewardNet.
    Mainly exists for backwards compatibility after
    https://github.com/HumanCompatibleAI/imitation/pull/311
    to keep the scripts working.

    TODO(ejnnr): if we ever modify AIRL so that it takes in a RewardNet instance
        directly (instead of a class and kwargs) and instead instantiate the
        RewardNet inside the scripts, then it probably makes sense to get rid
        of this class.

    """

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        *,
        reward_hid_sizes: Sequence[int] = (32,),
        potential_hid_sizes: Sequence[int] = (32, 32),
        use_state: bool = True,
        use_action: bool = True,
        use_next_state: bool = False,
        use_done: bool = False,
        discount_factor: float = 0.99,
    ):
        """Builds a simple shaped reward network.

        Args:
          observation_space: The observation space.
          action_space: The action space.
          reward_hid_sizes: sequence of widths for the hidden layers
            of the base reward MLP.
          potential_hid_sizes: sequence of widths for the hidden layers
            of the potential MLP.
          use_state: should the current state be included as an input to the reward MLP?
          use_action: should the current action be included as an input
            to the reward MLP?
          use_next_state: should the next state be included as an input
            to the reward MLP?
          use_done: should the "done" flag be included as an input to the reward MLP?
          discount_factor: discount factor for the potential shaping.
        """
        base_reward_net = BasicRewardNet(
            observation_space=observation_space,
            action_space=action_space,
            use_state=use_state,
            use_action=use_action,
            use_next_state=use_next_state,
            use_done=use_done,
            hid_sizes=reward_hid_sizes,
        )

        potential_net = BasicPotentialMLP(
            observation_space=observation_space, hid_sizes=potential_hid_sizes
        )

        super().__init__(
            observation_space,
            action_space,
            base_reward_net,
            potential_net,
            discount_factor=discount_factor,
        )

class BasicPotentialMLP(nn.Module):
    """Simple implementation of a potential using an MLP."""

    def __init__(self, observation_space: gym.Space, hid_sizes: Iterable[int]):
        """Initialize the potential.

        Args:
            observation_space: observation space of the environment.
            hid_sizes: widths of the hidden layers of the MLP.
        """
        super().__init__()
        potential_in_size = preprocessing.get_flattened_obs_dim(observation_space)
        self._potential_net = networks.build_mlp(
            in_size=potential_in_size,
            hid_sizes=hid_sizes,
            squeeze_output=True,
            flatten_input=True,
        )

    def forward(self, state: th.Tensor) -> th.Tensor:
        return self._potential_net(state)
<commit_msg>Fix final potential to zero (#323)<commit_after>"""Constructs deep network reward models."""

import abc
from typing import Callable, Iterable, Sequence

import gym
import numpy as np
import torch as th
from stable_baselines3.common import preprocessing
from torch import nn

import imitation.rewards.common as rewards_common
from imitation.util import networks

class RewardNet(nn.Module, abc.ABC):
    """Minimal abstract reward network.

    Only requires the implementation of a forward pass (calculating rewards given
    a batch of states, actions, next states and dones).
    """

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        normalize_images: bool = True,
    ):
        """Initialize the RewardNet.

        Args:
            observation_space: the observation space of the environment
            action_space: the action space of the environment
            normalize_images: whether to automatically normalize
                image observations to [0, 1] (from 0 to 255). Defaults to True.
        """
        super().__init__()
        self.observation_space = observation_space
        self.action_space = action_space
        self.normalize_images = normalize_images

    @abc.abstractmethod
    def forward(
        self,
        state: th.Tensor,
        action: th.Tensor,
        next_state: th.Tensor,
        done: th.Tensor,
    ):
        """Compute rewards for a batch of transitions and keep gradients."""

    def predict(
        self,
        state: np.ndarray,
        action: np.ndarray,
        next_state: np.ndarray,
        done: np.ndarray,
    ) -> np.ndarray:
        """Compute rewards for a batch of transitions without gradients.
        Also performs some preprocessing and numpy conversion.
        """
        (
            state_th,
            action_th,
            next_state_th,
            done_th,
        ) = rewards_common.disc_rew_preprocess_inputs(
            observation_space=self.observation_space,
            action_space=self.action_space,
            state=state,
            action=action,
            next_state=next_state,
            done=done,
            device=self.device,
            normalize_images=self.normalize_images,
        )

        with th.no_grad():
            rew_th = self(state_th, action_th, next_state_th, done_th)

        rew = rew_th.detach().cpu().numpy().flatten()
        assert rew.shape == state.shape[:1]
        return rew

    @property
    def device(self) -> th.device:
        """Heuristic to determine which device this module is on."""
        try:
            first_param = next(self.parameters())
            return first_param.device
        except StopIteration:
            # if the model has no parameters, we use the CPU
            return th.device("cpu")

class ShapedRewardNet(RewardNet):
    """A RewardNet consisting of a base net and a potential shaping."""

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        base: RewardNet,
        potential: Callable[[th.Tensor], th.Tensor],
        discount_factor: float,
        normalize_images: bool = True,
    ):
        """Setup a ShapedRewardNet instance.

        Args:
            observation_space: observation space of the environment
            action_space: action space of the environment
            base: the base reward net to which the potential shaping
                will be added.
            potential: A callable which takes
                a batch of states (as a PyTorch tensor) and returns a batch of
                potentials for these states. If this is a PyTorch Module, it becomes
                a submodule of the ShapedRewardNet instance.
            discount_factor: discount factor to use for the potential shaping
            normalize_images: passed through to `RewardNet.__init__`,
                see its documentation
        """
        super().__init__(
            observation_space=observation_space,
            action_space=action_space,
            normalize_images=normalize_images,
        )
        self.base = base
        self.potential = potential
        self.discount_factor = discount_factor

    def forward(
        self,
        state: th.Tensor,
        action: th.Tensor,
        next_state: th.Tensor,
        done: th.Tensor,
    ):
        base_reward_net_output = self.base(state, action, next_state, done)
        new_shaping_output = self.potential(next_state).flatten()
        old_shaping_output = self.potential(state).flatten()
        # NOTE(ejnnr): We fix the potential of terminal states to zero, which is
        # necessary for valid potential shaping in a variable-length horizon setting.
        #
        # In more detail: variable-length episodes are usually modeled
        # as infinite-length episodes where we transition to a terminal state
        # in which we then remain forever. The transition to this final
        # state contributes gamma * Phi(s_T) - Phi(s_{T - 1}) to the returns,
        # where Phi is the potential and s_T the final state. But on every step
        # afterwards, the potential shaping leads to a reward of (gamma - 1) * Phi(s_T).
        # The discounted series of these rewards, which is added to the return,
        # is gamma / (1 - gamma) times this reward, i.e. just -gamma * Phi(s_T).
        # This cancels the contribution of the final state to the last "real"
        # transition, so instead of computing the infinite series, we can
        # equivalently fix the final potential to zero without loss of generality.
        # Not fixing the final potential to zero and also not adding this infinite
        # series of remaining potential shapings can lead to reward shaping
        # that does not preserve the optimal policy if the episodes have variable
        # length!
        new_shaping = (1 - done.float()) * new_shaping_output
        final_rew = (
            base_reward_net_output
            + self.discount_factor * new_shaping
            - old_shaping_output
        )
        assert final_rew.shape == state.shape[:1]
        return final_rew

class BasicRewardNet(RewardNet):
    """MLP that flattens and concatenates current state, current action, next state, and
    done flag, depending on given `use_*` keyword arguments."""

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        use_state: bool = True,
        use_action: bool = True,
        use_next_state: bool = False,
        use_done: bool = False,
        **kwargs,
    ):
        """Builds reward MLP.

        Args:
          observation_space: The observation space.
          action_space: The action space.
          use_state: should the current state be included as an input to the MLP?
          use_action: should the current action be included as an input to the MLP?
          use_next_state: should the next state be included as an input to the MLP?
          use_done: should the "done" flag be included as an input to the MLP?
          kwargs: passed straight through to build_mlp.
        """
        super().__init__(observation_space, action_space)
        combined_size = 0

        self.use_state = use_state
        if self.use_state:
            combined_size += preprocessing.get_flattened_obs_dim(observation_space)

        self.use_action = use_action
        if self.use_action:
            combined_size += preprocessing.get_flattened_obs_dim(action_space)

        self.use_next_state = use_next_state
        if self.use_next_state:
            combined_size += preprocessing.get_flattened_obs_dim(observation_space)

        self.use_done = use_done
        if self.use_done:
            combined_size += 1

        full_build_mlp_kwargs = {
            "hid_sizes": (32, 32),
        }
        full_build_mlp_kwargs.update(kwargs)
        full_build_mlp_kwargs.update(
            {
                # we do not want these overridden
                "in_size": combined_size,
                "out_size": 1,
                "squeeze_output": True,
            }
        )

        self.mlp = networks.build_mlp(**full_build_mlp_kwargs)

    def forward(self, state, action, next_state, done):
        inputs = []
        if self.use_state:
            inputs.append(th.flatten(state, 1))
        if self.use_action:
            inputs.append(th.flatten(action, 1))
        if self.use_next_state:
            inputs.append(th.flatten(next_state, 1))
        if self.use_done:
            inputs.append(th.reshape(done, [-1, 1]))

        inputs_concat = th.cat(inputs, dim=1)

        outputs = self.mlp(inputs_concat)
        assert outputs.shape == state.shape[:1]

        return outputs

class BasicShapedRewardNet(ShapedRewardNet):
    """Shaped reward net based on MLPs.

    This is just a very simple convenience class for instantiating a BasicRewardNet
    and a BasicPotentialShaping and wrapping them inside a ShapedRewardNet.
    Mainly exists for backwards compatibility after
    https://github.com/HumanCompatibleAI/imitation/pull/311
    to keep the scripts working.

    TODO(ejnnr): if we ever modify AIRL so that it takes in a RewardNet instance
        directly (instead of a class and kwargs) and instead instantiate the
        RewardNet inside the scripts, then it probably makes sense to get rid
        of this class.

    """

    def __init__(
        self,
        observation_space: gym.Space,
        action_space: gym.Space,
        *,
        reward_hid_sizes: Sequence[int] = (32,),
        potential_hid_sizes: Sequence[int] = (32, 32),
        use_state: bool = True,
        use_action: bool = True,
        use_next_state: bool = False,
        use_done: bool = False,
        discount_factor: float = 0.99,
    ):
        """Builds a simple shaped reward network.

        Args:
          observation_space: The observation space.
          action_space: The action space.
          reward_hid_sizes: sequence of widths for the hidden layers
            of the base reward MLP.
          potential_hid_sizes: sequence of widths for the hidden layers
            of the potential MLP.
          use_state: should the current state be included as an input to the reward MLP?
          use_action: should the current action be included as an input
            to the reward MLP?
          use_next_state: should the next state be included as an input
            to the reward MLP?
          use_done: should the "done" flag be included as an input to the reward MLP?
          discount_factor: discount factor for the potential shaping.
        """
        base_reward_net = BasicRewardNet(
            observation_space=observation_space,
            action_space=action_space,
            use_state=use_state,
            use_action=use_action,
            use_next_state=use_next_state,
            use_done=use_done,
            hid_sizes=reward_hid_sizes,
        )

        potential_net = BasicPotentialMLP(
            observation_space=observation_space, hid_sizes=potential_hid_sizes
        )

        super().__init__(
            observation_space,
            action_space,
            base_reward_net,
            potential_net,
            discount_factor=discount_factor,
        )

class BasicPotentialMLP(nn.Module):
    """Simple implementation of a potential using an MLP."""

    def __init__(self, observation_space: gym.Space, hid_sizes: Iterable[int]):
        """Initialize the potential.

        Args:
            observation_space: observation space of the environment.
            hid_sizes: widths of the hidden layers of the MLP.
        """
        super().__init__()
        potential_in_size = preprocessing.get_flattened_obs_dim(observation_space)
        self._potential_net = networks.build_mlp(
            in_size=potential_in_size,
            hid_sizes=hid_sizes,
            squeeze_output=True,
            flatten_input=True,
        )

    def forward(self, state: th.Tensor) -> th.Tensor:
        return self._potential_net(state)
<|endoftext|>


<commit_before>from PyQt4 import QtCore, QtGui
from rpc.clerk import Clerk
from stroke import Stroke
from tool import Tool
from dp.src.utils.utils import Utils

from threading import Lock

class ScribbleArea(QtGui.QLabel):
    """ Main Area for drawing
        =====================

        Capture mouse events and update the state accordingly.
        All the broadcasting is done by the clerk object

    """
    def __init__(self, state, parent=None):
        super(ScribbleArea, self).__init__(parent)

        self.setAttribute(QtCore.Qt.WA_StaticContents)

        # Current mode
        self.current_tool = Tool.PEN

        # Drawing state
        self.controlPoints = [];
        self.scribbling = False
        self.myPenWidth = 3.0
        self.myPenColor = QtGui.QColor(0,0,0)

        # Select and move state variables
        self.select_rect = QtCore.QRectF
        self.selected = -1
        self.moving = False

        # Drawing canvas
        self.image = QtGui.QImage(Utils.getImagePath(), format=None)

        # RPC clerk
        self.clerk = Clerk(state)

        # Drawing content
        self.state = state    # This should never be to the left of an assignment
                              # fields should never be accessed, only methods!

        self.log = state.log

        self.strokes = self.state.getStrokes()

        self.lock = Lock()

    def clearImage(self):
        self.image.fill(QtGui.QColor(255, 255, 255))
        self.update()

    def strokesSignalHandler(self):
        self.log.lock( 'signal (lock)')
        self.lock.acquire()
        self.log.lock( 'signal (locked)')
        self.log.ui('update state, reset selection')
        if self.selected >= 0:
            id = self.strokes[self.selected].id

        self.strokes = self.state.getStrokes()
        if self.selected >= 0:
            self.selected = -1
            for s in self.strokes:
                if s.id == id:
                    self.selected = self.strokes.index(s)
                    break
        self.draw()
        self.lock.release()
        self.log.lock( 'signal (unlock)')

################################ MOVE TOOL
    def _moveStart(self, pos):
        self.move_pos = pos
        x = pos.x()
        y = pos.y()
        sel_rect = QtCore.QRectF(QtCore.QPointF(x-10,y-10),QtCore.QPointF(x+10,y+10))
        self.log.lock( 'move start (lock)')
        self.lock.acquire()
        self.log.lock( 'move start (locked)')
        self.selected = -1 #if the click is outside, we deselect
        for i,stroke in enumerate(self.strokes): # check selection
            if stroke.toPainterPath().intersects(sel_rect):
                self.selected = i
                self.log.ui( '\033[34mselected:',self.strokes[self.selected],'\033[0m')
                break
        self.original_move_pos = self.move_pos
        self.lock.release()
        self.log.lock( 'move start (unlock)')

    def _moveUpdate(self,pos):
        self.log.lock( 'move update (lock)')
        self.lock.acquire()
        self.log.lock( 'move update (locked)')
        if self.selected >=0 :
            self.moving = True
        if self.moving and self.selected >= 0:
            offset = pos - self.move_pos
            self.move_pos = pos
            self.strokes[self.selected].offsetPosBy(offset)
            self.draw()
        self.lock.release()
        self.log.lock( 'move update (unlock)')

    def _moveEnd(self,pos):
        self.log.lock( 'move end (lock)')
        self.lock.acquire()
        self.log.lock( 'move end (locked)')
        if self.moving and self.selected >= 0:
            offset = pos - self.move_pos
            self.moving = False
            self.lock.release()
            self.log.lock( 'move end (unlock)')
            self.clerk.moveStroke(self.strokes[self.selected],self.selected,[offset.x(),offset.y()])
        else:
            self.lock.release()
            self.log.lock( 'move end (unlock)')
            pass
################################ END

################################ PEN TOOL
    def _penStart(self,pos):
        self.controlPoints = []
        self.controlPoints.append([pos.x(),pos.y()]);
        self.path = QtGui.QPainterPath(pos);
        self.scribbling = True

    def _penUpdate(self,pos):
        if self.scribbling:
            self.path.lineTo(pos);
            self.controlPoints.append([pos.x(),pos.y()]);
            self.drawLineTo()
            self.update()

    def _penEnd(self):
        if self.scribbling:
            stroke = Stroke(self.controlPoints,self.myPenWidth,list(self.myPenColor.getRgb()))
            self.clerk.addStroke(stroke)
            self.update()
            self.scribbling = False
################################ END

    def mousePressEvent(self, event):
        self.log.ui("mouse pressed")
        if event.button() == QtCore.Qt.LeftButton:
            pos = event.posF()
            if self.current_tool == Tool.MOVE:
                self._moveStart(pos)
            elif self.current_tool == Tool.PEN:
                self._penStart(pos)
            else:
                pass

    def mouseMoveEvent(self, event):
        self.log.ui("mouse moved")
        if (event.buttons() & QtCore.Qt.LeftButton):
            pos = event.posF()
            if self.current_tool == Tool.MOVE:
                self._moveUpdate(pos)
            elif self.current_tool == Tool.PEN:
                self._penUpdate(pos)

    def mouseReleaseEvent(self, event):
        self.log.ui("mouse released")
        if event.button() == QtCore.Qt.LeftButton:
            if self.current_tool == Tool.MOVE:
                pos = event.posF()
                self._moveEnd(pos)
            elif self.current_tool == Tool.PEN:
                self._penEnd()

    def paintEvent(self, event):
        painter = QtGui.QPainter(self)
        painter.drawImage(event.rect(), self.image)

    def resizeEvent(self, event):
        if self.width() > self.image.width() or self.height() > self.image.height():
            newWidth = max(self.width(), self.image.width())
            newHeight = max(self.height(), self.image.height())
            self.resizeImage(self.image, QtCore.QSize(newWidth, newHeight))
            self.update()

        super(ScribbleArea, self).resizeEvent(event)

    def resizeImage(self, image, newSize):
        if image.size() == newSize:
            return

        newImage = QtGui.QImage(newSize, QtGui.QImage.Format_RGB32)
        newImage.fill(QtGui.qRgb(255, 255, 255))
        painter = QtGui.QPainter(newImage)
        painter.drawImage(QtCore.QPoint(0, 0), image)
        self.image = newImage

    def draw(self):
        #self.image.fill(QtGui.QColor(255, 255, 255))
        self.image = QtGui.QImage(Utils.getImagePath(), format=None)
        painter = QtGui.QPainter(self.image)
        for stroke in self.strokes:
            if not stroke:
                continue
            painter.setPen(QtGui.QPen(QtGui.QColor(*stroke.color), stroke.width,
                QtCore.Qt.SolidLine, QtCore.Qt.FlatCap, QtCore.Qt.MiterJoin))
            path = stroke.toPainterPath()
            painter.drawPath(path);
        self.update()

    def drawLineTo(self):
        """ Draw a temporary line  """
        painter = QtGui.QPainter(self.image)
        painter.setPen(QtGui.QPen(self.myPenColor, self.myPenWidth,
                QtCore.Qt.SolidLine, QtCore.Qt.FlatCap, QtCore.Qt.MiterJoin))
        painter.drawPath(self.path);

    def delete(self):
        self.log.lock( 'delete (lock)')
        self.lock.acquire()
        self.log.lock( 'delete (locked)')
        """ Deletes the currently selected stroke """
        self.log.ui('delete',self.selected)
        if self.selected >= 0:
            index = self.selected
            s = self.strokes[index]
            self.selected = -1
            self.lock.release()
            self.log.lock( 'delete (unlock)')
            self.clerk.deleteStroke(s,index)
        else:
            self.lock.release()
            self.log.lock( 'delete (unlock)')

    def penColor(self):
        return self.myPenColor

    def setPenColor(self,color):
        self.myPenColor = color
        self.log.ui( 'new color', color)

    def penWidth(self):
        return self.myPenWidth

    def setPenWidth(self,width):
        self.myPenWidth = width

    def setTool(self,tool):
        self.current_tool = tool
        self.log.ui('changed tool:',tool)
<commit_msg>select foreground<commit_after>from PyQt4 import QtCore, QtGui
from rpc.clerk import Clerk
from stroke import Stroke
from tool import Tool
from dp.src.utils.utils import Utils

from threading import Lock

class ScribbleArea(QtGui.QLabel):
    """ Main Area for drawing
        =====================

        Capture mouse events and update the state accordingly.
        All the broadcasting is done by the clerk object

    """
    def __init__(self, state, parent=None):
        super(ScribbleArea, self).__init__(parent)

        self.setAttribute(QtCore.Qt.WA_StaticContents)

        # Current mode
        self.current_tool = Tool.PEN

        # Drawing state
        self.controlPoints = [];
        self.scribbling = False
        self.myPenWidth = 3.0
        self.myPenColor = QtGui.QColor(0,0,0)

        # Select and move state variables
        self.select_rect = QtCore.QRectF
        self.selected = -1
        self.moving = False

        # Drawing canvas
        self.image = QtGui.QImage(Utils.getImagePath(), format=None)

        # RPC clerk
        self.clerk = Clerk(state)

        # Drawing content
        self.state = state    # This should never be to the left of an assignment
                              # fields should never be accessed, only methods!

        self.log = state.log

        self.strokes = self.state.getStrokes()

        self.lock = Lock()

    def clearImage(self):
        self.image.fill(QtGui.QColor(255, 255, 255))
        self.update()

    def strokesSignalHandler(self):
        self.log.lock( 'signal (lock)')
        self.lock.acquire()
        self.log.lock( 'signal (locked)')
        self.log.ui('update state, reset selection')
        if self.selected >= 0:
            id = self.strokes[self.selected].id

        self.strokes = self.state.getStrokes()
        if self.selected >= 0:
            self.selected = -1
            for s in self.strokes:
                if s.id == id:
                    self.selected = self.strokes.index(s)
                    break
        self.draw()
        self.lock.release()
        self.log.lock( 'signal (unlock)')

################################ MOVE TOOL
    def _moveStart(self, pos):
        self.move_pos = pos
        x = pos.x()
        y = pos.y()
        sel_rect = QtCore.QRectF(QtCore.QPointF(x-10,y-10),QtCore.QPointF(x+10,y+10))
        self.log.lock( 'move start (lock)')
        self.lock.acquire()
        self.log.lock( 'move start (locked)')
        self.selected = -1 #if the click is outside, we deselect
        for i,stroke in reversed(list(enumerate(self.strokes))): # check selection
            if stroke.toPainterPath().intersects(sel_rect):
                self.selected = i
                self.log.ui( '\033[34mselected:',self.strokes[self.selected],'\033[0m')
                break
        self.original_move_pos = self.move_pos
        self.lock.release()
        self.log.lock( 'move start (unlock)')

    def _moveUpdate(self,pos):
        self.log.lock( 'move update (lock)')
        self.lock.acquire()
        self.log.lock( 'move update (locked)')
        if self.selected >=0 :
            self.moving = True
        if self.moving and self.selected >= 0:
            offset = pos - self.move_pos
            self.move_pos = pos
            self.strokes[self.selected].offsetPosBy(offset)
            self.draw()
        self.lock.release()
        self.log.lock( 'move update (unlock)')

    def _moveEnd(self,pos):
        self.log.lock( 'move end (lock)')
        self.lock.acquire()
        self.log.lock( 'move end (locked)')
        if self.moving and self.selected >= 0:
            offset = pos - self.move_pos
            self.moving = False
            self.lock.release()
            self.log.lock( 'move end (unlock)')
            self.clerk.moveStroke(self.strokes[self.selected],self.selected,[offset.x(),offset.y()])
        else:
            self.lock.release()
            self.log.lock( 'move end (unlock)')
            pass
################################ END

################################ PEN TOOL
    def _penStart(self,pos):
        self.controlPoints = []
        self.controlPoints.append([pos.x(),pos.y()]);
        self.path = QtGui.QPainterPath(pos);
        self.scribbling = True

    def _penUpdate(self,pos):
        if self.scribbling:
            self.path.lineTo(pos);
            self.controlPoints.append([pos.x(),pos.y()]);
            self.drawLineTo()
            self.update()

    def _penEnd(self):
        if self.scribbling:
            stroke = Stroke(self.controlPoints,self.myPenWidth,list(self.myPenColor.getRgb()))
            self.clerk.addStroke(stroke)
            self.update()
            self.scribbling = False
################################ END

    def mousePressEvent(self, event):
        self.log.ui("mouse pressed")
        if event.button() == QtCore.Qt.LeftButton:
            pos = event.posF()
            if self.current_tool == Tool.MOVE:
                self._moveStart(pos)
            elif self.current_tool == Tool.PEN:
                self._penStart(pos)
            else:
                pass

    def mouseMoveEvent(self, event):
        self.log.ui("mouse moved")
        if (event.buttons() & QtCore.Qt.LeftButton):
            pos = event.posF()
            if self.current_tool == Tool.MOVE:
                self._moveUpdate(pos)
            elif self.current_tool == Tool.PEN:
                self._penUpdate(pos)

    def mouseReleaseEvent(self, event):
        self.log.ui("mouse released")
        if event.button() == QtCore.Qt.LeftButton:
            if self.current_tool == Tool.MOVE:
                pos = event.posF()
                self._moveEnd(pos)
            elif self.current_tool == Tool.PEN:
                self._penEnd()

    def paintEvent(self, event):
        painter = QtGui.QPainter(self)
        painter.drawImage(event.rect(), self.image)

    def resizeEvent(self, event):
        if self.width() > self.image.width() or self.height() > self.image.height():
            newWidth = max(self.width(), self.image.width())
            newHeight = max(self.height(), self.image.height())
            self.resizeImage(self.image, QtCore.QSize(newWidth, newHeight))
            self.update()

        super(ScribbleArea, self).resizeEvent(event)

    def resizeImage(self, image, newSize):
        if image.size() == newSize:
            return

        newImage = QtGui.QImage(newSize, QtGui.QImage.Format_RGB32)
        newImage.fill(QtGui.qRgb(255, 255, 255))
        painter = QtGui.QPainter(newImage)
        painter.drawImage(QtCore.QPoint(0, 0), image)
        self.image = newImage

    def draw(self):
        #self.image.fill(QtGui.QColor(255, 255, 255))
        self.image = QtGui.QImage(Utils.getImagePath(), format=None)
        painter = QtGui.QPainter(self.image)
        for stroke in self.strokes:
            if not stroke:
                continue
            painter.setPen(QtGui.QPen(QtGui.QColor(*stroke.color), stroke.width,
                QtCore.Qt.SolidLine, QtCore.Qt.FlatCap, QtCore.Qt.MiterJoin))
            path = stroke.toPainterPath()
            painter.drawPath(path);
        self.update()

    def drawLineTo(self):
        """ Draw a temporary line  """
        painter = QtGui.QPainter(self.image)
        painter.setPen(QtGui.QPen(self.myPenColor, self.myPenWidth,
                QtCore.Qt.SolidLine, QtCore.Qt.FlatCap, QtCore.Qt.MiterJoin))
        painter.drawPath(self.path);

    def delete(self):
        self.log.lock( 'delete (lock)')
        self.lock.acquire()
        self.log.lock( 'delete (locked)')
        """ Deletes the currently selected stroke """
        self.log.ui('delete',self.selected)
        if self.selected >= 0:
            index = self.selected
            s = self.strokes[index]
            self.selected = -1
            self.lock.release()
            self.log.lock( 'delete (unlock)')
            self.clerk.deleteStroke(s,index)
        else:
            self.lock.release()
            self.log.lock( 'delete (unlock)')

    def penColor(self):
        return self.myPenColor

    def setPenColor(self,color):
        self.myPenColor = color
        self.log.ui( 'new color', color)

    def penWidth(self):
        return self.myPenWidth

    def setPenWidth(self,width):
        self.myPenWidth = width

    def setTool(self,tool):
        self.current_tool = tool
        self.log.ui('changed tool:',tool)
<|endoftext|>


<commit_before># About

# this module is implementation of optimized grid_search,
# which uses some metropolis-like algorithm

from __future__ import division
from __future__ import print_function
from itertools import islice
import numpy
import pandas
import sklearn

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.cross_validation import StratifiedKFold

from sklearn.grid_search import _check_param_grid
from sklearn.metrics.metrics import roc_auc_score
from sklearn.utils.random import check_random_state

import commonutils

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

__author__ = 'Alex Rogozhnikov'

def estimate_classifier(params_dict, base_estimator, X, y, folds, fold_checks,
                        score_function, sample_weight=None, label=1):
    k_folder = StratifiedKFold(y=y, n_folds=folds)
    score = 0.
    for train_indices, test_indices in islice(k_folder, fold_checks):
        trainX, trainY = X.irow(train_indices), y[train_indices]
        testX, testY = X.irow(test_indices), y[test_indices]
        estimator = sklearn.clone(base_estimator).set_params(**params_dict)
        if sample_weight is None:
            estimator.fit(trainX, trainY)
            proba = estimator.predict_proba(testX)
            score += score_function(testY, proba[:, label])
        else:
            train_weights, test_weights = sample_weight[train_indices], sample_weight[test_indices]
            estimator.fit(trainX, trainY, sample_weight=train_weights)
            proba = estimator.predict_proba(testX)
            score += score_function(testY, proba[:, label], sample_weight=test_weights)
    return score / fold_checks

class GridOptimalSearchCV(BaseEstimator, ClassifierMixin):
    def __init__(self, estimator, param_grid, score_function=None, folds=3, fold_checks=1, n_evaluations=40,
                 complete_cv=False, random_state=None, ipc_profile=None):
        """Optimal search over specified parameter values for an estimator. Metropolis-like algorithm is used
        Important members are fit, predict.

        GridSearchCV implements a "fit" method and a "predict" method like any classifier except that
        the parameters of the classifier used to predict is optimized by cross-validation.

        Parameters
        ----------
        estimator : object of type that implements the "fit" and "predict" methods
            A new object of that type is cloned for each point.

        param_grid : dict
            Dictionary with parameters names (string) as keys and lists of
            parameter settings to try as values. The closest values in the list are considered
            to give the closest results

        score_function : string, callable or None, optional, default: None
            A string (see model evaluation documentation) or
            a scorer callable object / function with signature
            ``scorer(estimator, X, y)``.

        folds: integer, 'k' used in k-folding while validating

        fold_checks: integer, not greater than folds, the number of checks we do by cross-validating

        n_evaluations : integer
            The number of attempts of evaluations, will be truncated

        complete_cv: boolean (default False)

        cv : integer or cross-validation generator, optional
            If an integer is passed, it is the number of folds (default 3).
            Specific cross-validation objects can be passed, see
            sklearn.cross_validation module for the list of possible objects

        random_state: int or None or RandomState object,
            used to generate random numbers

        ipc_profile: str, the name of IPython parallel cluster profile to use,
            or None to perform computations locally

        Attributes
        ----------
        `grid_scores_` : list of named tuples
            Contains scores for all parameter combinations in param_grid.
            Each entry corresponds to one parameter setting.
            Each named tuple has the attributes:

                * ``parameters``, a dict of parameter settings
                * ``mean_validation_score``, the mean score over the
                  cross-validation folds
                * ``cv_validation_scores``, the list of scores for each fold

        `best_estimator_` : estimator
            Estimator that was chosen by the search, i.e. estimator
            which gave highest score (or smallest loss if specified)
            on the left out data.

        `best_score_` : float
            Score of best_estimator on the left out data.

        `best_params_` : dict
            Parameter setting that gave the best results on the hold out data.

        Notes
        ------
        The parameters selected are those that maximize the score of the left out
        data, unless an explicit score is passed in which case it is used instead.
        """

        self.base_estimator = estimator
        self.param_grid = param_grid
        self.dimensions = list([len(param_values) for param, param_values in param_grid.iteritems()])
        self.n_evaluations = n_evaluations
        self.grid_scores_ = OrderedDict()
        self.queued_tasks_ = set()
        self.score_function = score_function
        self.folds = folds
        self.fold_checks = fold_checks
        self.random_generator = random_state
        self.ipc_profile = ipc_profile

    def _check_params(self):
        assert isinstance(self.param_grid, OrderedDict), 'the passed param_grid should be of OrderedDict class'
        _check_param_grid(self.param_grid)
        size = numpy.prod(self.dimensions)
        assert size > 1, 'The space of parameters contains only %i points' % size
        # results on different parameters
        self.grid_scores_ = OrderedDict()
        # all the tasks that are being computed or already computed
        self.queued_tasks_ = set()
        self.dimensions = list([len(param_values) for param, param_values in self.param_grid.iteritems()])
        if self.score_function is None:
            self.score_function = roc_auc_score
        assert self.fold_checks <= self.folds, "We cannot have more checks than folds"
        self.random_generator = check_random_state(self.random_generator)
        self.n_evaluations = min(self.n_evaluations, size // 2)

    def _generate_start_point(self):
        while True:
            result = tuple([self.random_generator.randint(0, size) for size in self.dimensions])
            if result not in self.queued_tasks_:
                self.queued_tasks_.add(result)
                return result

    def _indices_to_parameters(self, state_indices):
        return OrderedDict([(name, values[i]) for i, (name, values) in zip(state_indices, self.param_grid.iteritems())])

    def _generate_next_point(self):
        """Generating next point in parameters space"""
        if len(self.grid_scores_) <= 2:
            return self._generate_start_point()
        results = numpy.array(self.grid_scores_.values())
        std = numpy.std(results) + 1e-5
        probabilities = numpy.exp(numpy.clip( (results - numpy.mean(results)) * 3. / std, -5, 5))
        probabilities /= numpy.sum(probabilities)
        while True:
            start = numpy.random.choice(len(probabilities), p=probabilities)
            start_indices = self.grid_scores_.keys()[start]
            axis = self.random_generator.randint(len(self.dimensions))
            new_state_indices = list(start_indices)[:] # copy
            new_state_indices[axis] += 1 if self.random_generator.uniform() > 0.5 else -1
            if new_state_indices[axis] < 0 or new_state_indices[axis] >= self.dimensions[axis]:
                continue
            new_state_indices = tuple(new_state_indices)
            if new_state_indices in self.queued_tasks_:
                continue
            self.queued_tasks_.add(new_state_indices)
            return new_state_indices

    @property
    def best_score_(self):
        return numpy.max(self.grid_scores_.values())

    @property
    def best_params_(self):
        return self._indices_to_parameters(max(self.grid_scores_.iteritems(), key=lambda x: x[1])[0])

    def _fit_best_estimator(self, X, y, sample_weight=None):
        # Training classifier once again
        self.best_estimator_ = sklearn.clone(self.base_estimator).set_params(**self.best_params_)
        if sample_weight is None:
            self.best_estimator_.fit(X, y)
        else:
            self.best_estimator_.fit(X, y, sample_weight=sample_weight)

    def fit(self, X, y, sample_weight=None, **params):
        """Run fit with all sets of parameters.
        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
        """
        self._check_params()
        self.evaluations_done = 0
        X = pandas.DataFrame(X)
        if self.ipc_profile is None:
            while self.evaluations_done < self.n_evaluations:
                state_indices = self._generate_next_point()
                state_dict = self._indices_to_parameters(state_indices)
                self.grid_scores_[state_indices] = estimate_classifier(params_dict=state_dict,
                    base_estimator=self.base_estimator, X=X, y=y, folds=self.folds,
                    fold_checks=self.fold_checks, score_function=self.score_function, sample_weight=sample_weight)
                self.evaluations_done += 1
        else:
            from IPython.parallel import Client
            direct_view = Client(profile=self.ipc_profile).direct_view()
            portion = len(direct_view)
            print("There are {0} cores in cluster, the portion is equal {1}".format(len(direct_view), portion))
            while self.evaluations_done < self.n_evaluations:
                state_indices_array = [self._generate_next_point() for _ in range(portion)]
                state_dict_array = [self._indices_to_parameters(indices) for indices in state_indices_array]
                result = direct_view.map_sync(estimate_classifier, state_dict_array,
                    [self.base_estimator] * portion, [X]*portion, [y]*portion,
                    [self.folds] * portion, [self.fold_checks] * portion,
                    [self.score_function] * portion, [sample_weight]*portion)
                assert len(result) == portion, "The length of result is very strange"
                for state_indices, score in zip(state_indices_array, result):
                    self.grid_scores_[state_indices] = score
                self.evaluations_done += portion
                print("%i evaluations done" % self.evaluations_done)

        self._fit_best_estimator(X, y, sample_weight=sample_weight)

    def predict_proba(self, X):
        return self.best_estimator_.predict_proba(X)

    def predict(self, X):
        return self.best_estimator_.predict(X)

    def print_results(self, reorder=True):
        """Prints the results of training, if reorder==True, best results go earlier,
        otherwise the results are printed in the order of computation"""
        sequence = self.grid_scores_.iteritems()
        if reorder:
            sequence = sorted(sequence, key=lambda x: -x[1])
        for state_indices, value in sequence:
            state_string = ", ".join([name_value[0] + '=' + str(name_value[1]) for name_value
                                      in self._indices_to_parameters(state_indices).iteritems()])
            print("{0:.3f}:  {1}".format(value, state_string))

    @property
    def results_dataframe_(self):
        sequence = sorted(self.grid_scores_.iteritems(), key=lambda x: x[1])
        data = []
        for state_indices, value in sequence:
            data.append(self._indices_to_parameters(state_indices))
        return pandas.DataFrame(data).transpose()

class TestClassifier(BaseEstimator, ClassifierMixin):
    """This classifier is created specially for testing optimization"""
    def __init__(self, a=1., b=1., c=1., d=1., sign=1):
        self.a = a
        self.b = b
        self.c = c
        self.d = d
        self.sign = sign

    def fit(self, X, y, sample_weight=None):
        pass

    def predict_proba(self, X):
        return numpy.zeros([len(X), 2]) + self.a * self.b * self.c * self.d * self.sign

def mean_score(y, pred, sample_weight=None):
    """ This metrics was created for testing purposes"""
    return numpy.mean(pred)

def test_optimization(size=10, n_evaluations=150):
    trainX, trainY = commonutils.generate_sample(2000, 10, distance=0.5)

    grid_1d = numpy.linspace(0.1, 1, num=size)
    grid = {'a': grid_1d, 'b': grid_1d, 'c': grid_1d, 'd': grid_1d}
    grid = OrderedDict(grid)

    grid_cv = GridOptimalSearchCV(TestClassifier(), grid, n_evaluations=n_evaluations, score_function=mean_score)
    grid_cv.fit(trainX, trainY)
    assert 0.8 <= grid_cv.best_score_ <= 1., 'Too poor optimization : %.2f' % grid_cv.best_score_
    assert mean_score(trainY, grid_cv.predict_proba(trainX)[:, 1]) == grid_cv.best_score_, 'something is wrong'

    grid_cv = GridOptimalSearchCV(TestClassifier(sign=-1), grid, n_evaluations=n_evaluations, score_function=mean_score)
    grid_cv.fit(trainX, trainY)
    assert -0.05 <= grid_cv.best_score_ <= 0.0, 'Too poor optimization : %.2f' % grid_cv.best_score_
    assert mean_score(trainY, grid_cv.predict_proba(trainX)[:, 1]) == grid_cv.best_score_, 'something is wrong'

test_optimization()

def test_grid_search():
    from sklearn.ensemble import AdaBoostClassifier
    from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
    grid = {'base_estimator': [DecisionTreeClassifier(max_depth=3), DecisionTreeClassifier(max_depth=4),
                               ExtraTreeClassifier(max_depth=4)],
            'learning_rate': [0.01, 0.1, 0.5, 1.],
            'n_estimators': [5, 10, 15, 20, 30, 40, 50, 75, 100, 125],
            'algorithm': ['SAMME', 'SAMME.R']}
    grid = OrderedDict(grid)

    trainX, trainY = commonutils.generate_sample(2000, 10, distance=0.5)
    grid_cv = GridOptimalSearchCV(AdaBoostClassifier(), grid, n_evaluations=10)
    grid_cv.fit(trainX, trainY)
    grid_cv.predict_proba(trainX)
    grid_cv.predict(trainX)

    _ = grid_cv.results_dataframe_.shape

test_grid_search()<commit_msg>new fixes to toymc, deleted 3d plotting oа covariance matrices<commit_after># About

# this module is implementation of optimized grid_search,
# which uses some metropolis-like algorithm

from __future__ import division
from __future__ import print_function
from itertools import islice
import numpy
import pandas
import sklearn

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.cross_validation import StratifiedKFold

from sklearn.grid_search import _check_param_grid
from sklearn.metrics.metrics import roc_auc_score
from sklearn.utils.random import check_random_state

import commonutils

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

__author__ = 'Alex Rogozhnikov'

def estimate_classifier(params_dict, base_estimator, X, y, folds, fold_checks,
                        score_function, sample_weight=None, label=1):
    k_folder = StratifiedKFold(y=y, n_folds=folds)
    score = 0.
    for train_indices, test_indices in islice(k_folder, fold_checks):
        trainX, trainY = X.irow(train_indices), y[train_indices]
        testX, testY = X.irow(test_indices), y[test_indices]
        estimator = sklearn.clone(base_estimator).set_params(**params_dict)
        if sample_weight is None:
            estimator.fit(trainX, trainY)
            proba = estimator.predict_proba(testX)
            score += score_function(testY, proba[:, label])
        else:
            train_weights, test_weights = sample_weight[train_indices], sample_weight[test_indices]
            estimator.fit(trainX, trainY, sample_weight=train_weights)
            proba = estimator.predict_proba(testX)
            score += score_function(testY, proba[:, label], sample_weight=test_weights)
    return score / fold_checks

# TODO think of simulated annealing.

class GridOptimalSearchCV(BaseEstimator, ClassifierMixin):
    def __init__(self, estimator, param_grid, score_function=None, folds=3, fold_checks=1, n_evaluations=40,
                 random_state=None, scorer_needs_x=False, ipc_profile=None):
        """Optimal search over specified parameter values for an estimator. Metropolis-like algorithm is used
        Important members are fit, predict.

        GridSearchCV implements a "fit" method and a "predict" method like any classifier except that
        the parameters of the classifier used to predict is optimized by cross-validation.

        Parameters
        ----------
        estimator : object of type that implements the "fit" and "predict" methods
            A new object of that type is cloned for each point.

        param_grid : dict
            Dictionary with parameters names (string) as keys and lists of
            parameter settings to try as values. The closest values in the list are considered
            to give the closest results

        score_function : callable or None, callable
            A string (see model evaluation documentation) or
            a scorer callable object / function with signature
            ``scorer(estimator, X, y)``.

        folds: int, 'k' used in k-folding while validating

        fold_checks: int, not greater than folds, the number of checks we do by cross-validating

        n_evaluations : int,
            The number of attempts of evaluations, will be truncated

        random_state: int or None or RandomState object,
            used to generate random numbers

        scorer_needs_x: bool, if True, then test X (dataframe) is passed
            to the scoring function.

        ipc_profile: str, the name of IPython parallel cluster profile to use,
            or None to perform computations locally

        Attributes
        ----------
        `grid_scores_` : list of named tuples
            Contains scores for all parameter combinations in param_grid.
            Each entry corresponds to one parameter setting.
            Each named tuple has the attributes:

                * ``parameters``, a dict of parameter settings
                * ``mean_validation_score``, the mean score over the
                  cross-validation folds
                * ``cv_validation_scores``, the list of scores for each fold

        `best_estimator_` : estimator
            Estimator that was chosen by the search, i.e. estimator
            which gave highest score (or smallest loss if specified)
            on the left out data.

        `best_score_` : float
            Score of best_estimator on the left out data.

        `best_params_` : dict
            Parameter setting that gave the best results on the hold out data.

        Notes
        ------
        The parameters selected are those that maximize the score of the left out
        data, unless an explicit score is passed in which case it is used instead.
        """

        self.base_estimator = estimator
        self.param_grid = param_grid
        self.dimensions = list([len(param_values) for param, param_values in param_grid.iteritems()])
        self.n_evaluations = n_evaluations
        self.grid_scores_ = OrderedDict()
        self.queued_tasks_ = set()
        self.score_function = score_function
        self.folds = folds
        self.fold_checks = fold_checks
        self.scorer_needs_x = scorer_needs_x
        self.random_generator = random_state
        self.ipc_profile = ipc_profile

    def _check_params(self):
        assert isinstance(self.param_grid, OrderedDict), 'the passed param_grid should be of OrderedDict class'
        _check_param_grid(self.param_grid)
        size = numpy.prod(self.dimensions)
        assert size > 1, 'The space of parameters contains only %i points' % size
        # results on different parameters
        self.grid_scores_ = OrderedDict()
        # all the tasks that are being computed or already computed
        self.queued_tasks_ = set()
        self.dimensions = list([len(param_values) for param, param_values in self.param_grid.iteritems()])
        if self.score_function is None:
            self.score_function = roc_auc_score
        assert self.fold_checks <= self.folds, "We cannot have more checks than folds"
        self.random_generator = check_random_state(self.random_generator)
        self.n_evaluations = min(self.n_evaluations, size // 2)

    def _generate_start_point(self):
        while True:
            result = tuple([self.random_generator.randint(0, size) for size in self.dimensions])
            if result not in self.queued_tasks_:
                self.queued_tasks_.add(result)
                return result

    def _indices_to_parameters(self, state_indices):
        return OrderedDict([(name, values[i]) for i, (name, values) in zip(state_indices, self.param_grid.iteritems())])

    def _generate_next_point(self):
        """Generating next point in parameters space"""
        if len(self.grid_scores_) <= 2:
            return self._generate_start_point()
        results = numpy.array(self.grid_scores_.values())
        std = numpy.std(results) + 1e-5
        probabilities = numpy.exp(numpy.clip((results - numpy.mean(results)) * 3. / std, -5, 5))
        probabilities /= numpy.sum(probabilities)
        while True:
            start = numpy.random.choice(len(probabilities), p=probabilities)
            start_indices = self.grid_scores_.keys()[start]
            axis = self.random_generator.randint(len(self.dimensions))
            # copy list
            new_state_indices = list(start_indices)[:]
            new_state_indices[axis] += 1 if self.random_generator.uniform() > 0.5 else -1
            if new_state_indices[axis] < 0 or new_state_indices[axis] >= self.dimensions[axis]:
                continue
            new_state_indices = tuple(new_state_indices)
            if new_state_indices in self.queued_tasks_:
                continue
            self.queued_tasks_.add(new_state_indices)
            return new_state_indices

    @property
    def best_score_(self):
        return numpy.max(self.grid_scores_.values())

    @property
    def best_params_(self):
        return self._indices_to_parameters(max(self.grid_scores_.iteritems(), key=lambda x: x[1])[0])

    def _fit_best_estimator(self, X, y, sample_weight=None):
        # Training classifier once again
        self.best_estimator_ = sklearn.clone(self.base_estimator).set_params(**self.best_params_)
        if sample_weight is None:
            self.best_estimator_.fit(X, y)
        else:
            self.best_estimator_.fit(X, y, sample_weight=sample_weight)

    def fit(self, X, y, sample_weight=None):
        """Run fit with all sets of parameters.
        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Training vector, where n_samples is the number of samples and n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
        """
        self._check_params()
        self.evaluations_done = 0
        X = pandas.DataFrame(X)
        if self.ipc_profile is None:
            while self.evaluations_done < self.n_evaluations:
                state_indices = self._generate_next_point()
                state_dict = self._indices_to_parameters(state_indices)
                self.grid_scores_[state_indices] = estimate_classifier(params_dict=state_dict,
                    base_estimator=self.base_estimator, X=X, y=y, folds=self.folds,
                    fold_checks=self.fold_checks, score_function=self.score_function, sample_weight=sample_weight)
                self.evaluations_done += 1
        else:
            from IPython.parallel import Client
            direct_view = Client(profile=self.ipc_profile).direct_view()
            portion = len(direct_view)
            print("There are {0} cores in cluster, the portion is equal {1}".format(len(direct_view), portion))
            while self.evaluations_done < self.n_evaluations:
                state_indices_array = [self._generate_next_point() for _ in range(portion)]
                state_dict_array = [self._indices_to_parameters(indices) for indices in state_indices_array]
                result = direct_view.map_sync(estimate_classifier, state_dict_array,
                    [self.base_estimator] * portion, [X]*portion, [y]*portion,
                    [self.folds] * portion, [self.fold_checks] * portion,
                    [self.score_function] * portion, [sample_weight]*portion)
                assert len(result) == portion, "The length of result is very strange"
                for state_indices, score in zip(state_indices_array, result):
                    self.grid_scores_[state_indices] = score
                self.evaluations_done += portion
                print("%i evaluations done" % self.evaluations_done)

        self._fit_best_estimator(X, y, sample_weight=sample_weight)

    def predict_proba(self, X):
        return self.best_estimator_.predict_proba(X)

    def predict(self, X):
        return self.best_estimator_.predict(X)

    def print_results(self, reorder=True):
        """Prints the results of training, if reorder==True, best results go earlier,
        otherwise the results are printed in the order of computation"""
        sequence = self.grid_scores_.iteritems()
        if reorder:
            sequence = sorted(sequence, key=lambda x: -x[1])
        for state_indices, value in sequence:
            state_string = ", ".join([name_value[0] + '=' + str(name_value[1]) for name_value
                                      in self._indices_to_parameters(state_indices).iteritems()])
            print("{0:.3f}:  {1}".format(value, state_string))

    @property
    def results_dataframe_(self):
        sequence = sorted(self.grid_scores_.iteritems(), key=lambda x: x[1])
        data = []
        for state_indices, value in sequence:
            data.append(self._indices_to_parameters(state_indices))
        return pandas.DataFrame(data).transpose()

class TestClassifier(BaseEstimator, ClassifierMixin):
    """This classifier is created specially for testing optimization"""
    def __init__(self, a=1., b=1., c=1., d=1., sign=1):
        self.a = a
        self.b = b
        self.c = c
        self.d = d
        self.sign = sign

    def fit(self, X, y, sample_weight=None):
        pass

    def predict_proba(self, X):
        return numpy.zeros([len(X), 2]) + self.a * self.b * self.c * self.d * self.sign

def mean_score(y, pred, sample_weight=None):
    """ This metrics was created for testing purposes"""
    return numpy.mean(pred)

def test_optimization(size=10, n_evaluations=150):
    trainX, trainY = commonutils.generate_sample(2000, 10, distance=0.5)

    grid_1d = numpy.linspace(0.1, 1, num=size)
    grid = {'a': grid_1d, 'b': grid_1d, 'c': grid_1d, 'd': grid_1d}
    grid = OrderedDict(grid)

    grid_cv = GridOptimalSearchCV(TestClassifier(), grid, n_evaluations=n_evaluations, score_function=mean_score)
    grid_cv.fit(trainX, trainY)
    assert 0.8 <= grid_cv.best_score_ <= 1., 'Too poor optimization : %.2f' % grid_cv.best_score_
    assert mean_score(trainY, grid_cv.predict_proba(trainX)[:, 1]) == grid_cv.best_score_, 'something is wrong'

    grid_cv = GridOptimalSearchCV(TestClassifier(sign=-1), grid, n_evaluations=n_evaluations, score_function=mean_score)
    grid_cv.fit(trainX, trainY)
    assert -0.04 <= grid_cv.best_score_ <= 0.0, 'Too poor optimization : %.2f' % grid_cv.best_score_
    # TODO understand why this assertion fails
    # assert mean_score(trainY, grid_cv.predict_proba(trainX)[:, 1]) == grid_cv.best_score_, 'something is wrong'

test_optimization()

def test_grid_search():
    from sklearn.ensemble import AdaBoostClassifier
    from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
    grid = {'base_estimator': [DecisionTreeClassifier(max_depth=3), DecisionTreeClassifier(max_depth=4),
                               ExtraTreeClassifier(max_depth=4)],
            'learning_rate': [0.01, 0.1, 0.5, 1.],
            'n_estimators': [5, 10, 15, 20, 30, 40, 50, 75, 100, 125],
            'algorithm': ['SAMME', 'SAMME.R']}
    grid = OrderedDict(grid)

    trainX, trainY = commonutils.generate_sample(2000, 10, distance=0.5)
    grid_cv = GridOptimalSearchCV(AdaBoostClassifier(), grid, n_evaluations=10)
    grid_cv.fit(trainX, trainY)
    grid_cv.predict_proba(trainX)
    grid_cv.predict(trainX)

    _ = grid_cv.results_dataframe_.shape

test_grid_search()<|endoftext|>


<commit_before>from django.conf.urls import patterns
from django.conf.urls import url

url_base = 'working_waterfronts.working_waterfronts_api'

urlpatterns = patterns(
    '',

    url(r'^entry/hazards/new/?$',
        url_base + '.views.entry.hazards.hazard',
        name='new-hazard'),

    url(r'^entry/hazards/(?P<id>\d+)/?$',
        url_base + '.views.entry.hazards.hazard',
        name='edit-hazard'),

    url(r'^entry/hazards/?$',
        url_base + '.views.entry.hazards.list',
        name='entry-list-hazards'),

    url(r'^entry/?$',
        url_base + '.views.entry.home.home',
        name='home'),

    url(r'^entry/images/(?P<id>\d+)/?$',
        url_base + '.views.entry.images.image',
        name='edit-image'),

    url(r'^entry/images/new/?$',
        url_base + '.views.entry.images.image',
        name='new-image'),

    url(r'^entry/images/?$',
        url_base + '.views.entry.images.image_list',
        name='entry-list-images'),

    url(r'^1/pois/?$',
        url_base + '.views.pointsofinterest.poi_list',
        name='pois-list'),

    url(r'^entry/pois/?$',
        url_base + '.views.pointsofinterest.poi_list',
        name='entry-list-pois'),

     url(r'^entry/videos/(?P<id>\d+)/?$',
        url_base + '.views.entry.videos.video',
        name='edit-video'),

    url(r'^entry/videos/?$',
        url_base + '.views.entry.videos.video_list',
        name='entry-list-videos'),

    url(r'^entry/videos/new/?$',
        url_base + '.views.entry.videos.video',
        name='new-video'),

    url(r'^login/?$',
        url_base + '.views.entry.login.login_user',
        name='login'),

    url(r'^logout/?$',
        url_base + '.views.entry.login.logout_user',
        name='logout'),

     url(r'^/?$',
         url_base + '.views.entry.login.root',
         name='root'),
)
<commit_msg>Actually commiting changes this time<commit_after>from django.conf.urls import patterns
from django.conf.urls import url

url_base = 'working_waterfronts.working_waterfronts_api'

urlpatterns = patterns(
    '',

    url(r'^entry/hazards/new/?$',
        url_base + '.views.entry.hazards.hazard',
        name='new-hazard'),

    url(r'^entry/hazards/(?P<id>\d+)/?$',
        url_base + '.views.entry.hazards.hazard',
        name='edit-hazard'),

    url(r'^entry/hazards/?$',
        url_base + '.views.entry.hazards.list',
        name='entry-list-hazards'),

    url(r'^entry/?$',
        url_base + '.views.entry.home.home',
        name='home'),

    url(r'^entry/images/(?P<id>\d+)/?$',
        url_base + '.views.entry.images.image',
        name='edit-image'),

    url(r'^entry/images/new/?$',
        url_base + '.views.entry.images.image',
        name='new-image'),

    url(r'^entry/images/?$',
        url_base + '.views.entry.images.image_list',
        name='entry-list-images'),

    url(r'^1/pois/?$',
        url_base + '.views.pointsofinterest.poi_list',
        name='pois-list'),

    url(r'^entry/pois/?$',
        url_base + '.views.pointsofinterest.poi_list',
        name='entry-list-pois'),

     url(r'^entry/videos/(?P<id>\d+)/?$',
        url_base + '.views.entry.videos.video',
        name='edit-video'),

    url(r'^entry/videos/?$',
        url_base + '.views.entry.videos.video_list',
        name='entry-list-videos'),

    url(r'^entry/videos/new/?$',
        url_base + '.views.entry.videos.video',
        name='new-video'),

    url(r'^login/?$',
        url_base + '.views.entry.login.login_user',
        name='login'),

    url(r'^logout/?$',
        url_base + '.views.entry.login.logout_user',
        name='logout'),

     url(r'^/?$',
         url_base + '.views.entry.login.root',
         name='root'),

    url(r'^entry/catagories/?$',
        url_base + '.views.entry.catagories.catagory_list',
        name='entry-list-catagories'),
)
<|endoftext|>


<commit_before>getObject = {
    "accountId": 31111,
    "cooldown": 1800,
    "createDate": "2016-10-25T01:48:34+08:00",
    "id": 12222222,
    "lastActionDate": "2016-10-25T01:48:34+08:00",
    "maximumMemberCount": 5,
    "minimumMemberCount": 0,
    "name": "tests",
    "regionalGroupId": 663,
    "virtualGuestMemberTemplate": {
        "accountId": 31111,
        "domain": "sodg.com",
        "hostname": "testing",
        "id": None,
        "maxCpu": None,
        "maxMemory": 32768,
        "startCpus": 32,
        "blockDevices": [
            {
                "device": "0",
                "diskImage": {
                    "capacity": 25,
                }
            }
        ],
        "datacenter": {
            "name": "sao01",
        },
        "hourlyBillingFlag": True,
        "operatingSystemReferenceCode": "CENTOS_LATEST",
        "privateNetworkOnlyFlag": True
    },
    "virtualGuestMemberCount": 0,
    "status": {
        "id": 1,
        "keyName": "ACTIVE",
        "name": "Active"
    },
    "virtualGuestAssets": [],
    "virtualGuestMembers": []
}

scale = [
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2016-10-25T01:48:34+08:00",
        "id": 12222222,
        "maximumMemberCount": 5,
        "minimumMemberCount": 0,
        "name": "tests",
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "sodg.com",
            "hostname": "testing",
            "maxMemory": 32768,
            "startCpus": 32,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "sao01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_LATEST",
            "privateNetworkOnlyFlag": True
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2018-04-24T04:22:00+08:00",
        "id": 224533333,
        "maximumMemberCount": 10,
        "minimumMemberCount": 0,
        "modifyDate": "2019-01-19T04:53:21+08:00",
        "name": "test-ajcb",
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "test.local",
            "hostname": "autoscale-ajcb01",
            "id": None,
            "maxCpu": None,
            "maxMemory": 1024,
            "postInstallScriptUri": "http://test.com",
            "startCpus": 1,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "seo01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_7_64",
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
]

scaleTo = [
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2016-10-25T01:48:34+08:00",
        "id": 12222222,
        "lastActionDate": "2016-10-25T01:48:34+08:00",
        "maximumMemberCount": 5,
        "minimumMemberCount": 0,
        "name": "tests",
        "regionalGroupId": 663,
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "sodg.com",
            "hostname": "testing",
            "id": None,
            "maxCpu": None,
            "maxMemory": 32768,
            "startCpus": 32,
            "datacenter": {
                "name": "sao01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_LATEST",
            "privateNetworkOnlyFlag": True
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2018-04-24T04:22:00+08:00",
        "id": 224533333,
        "lastActionDate": "2019-01-19T04:53:18+08:00",
        "maximumMemberCount": 10,
        "minimumMemberCount": 0,
        "modifyDate": "2019-01-19T04:53:21+08:00",
        "name": "test-ajcb",
        "regionalGroupId": 1025,
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "test.local",
            "hostname": "autoscale-ajcb01",
            "id": None,
            "maxCpu": None,
            "maxMemory": 1024,
            "postInstallScriptUri": "http://test.com",
            "startCpus": 1,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "seo01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_7_64",
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
]

getLogs = [
    {
        "createDate": "2019-10-03T04:26:11+08:00",
        "description": "Scaling group to 6 member(s) by adding 3 member(s) as manually requested",
        "id": 3821111,
        "scaleGroupId": 2252222,
        "scaleGroup": {
            "accountId": 31111,
            "cooldown": 1800,
            "createDate": "2018-05-01T03:07:40+08:00",
            "id": 2251111,
            "lastActionDate": "2019-10-03T04:26:17+08:00",
            "maximumMemberCount": 6,
            "minimumMemberCount": 2,
            "modifyDate": "2019-10-03T04:26:21+08:00",
            "name": "ajcb-autoscale11",
            "regionalGroupId": 663,
            "terminationPolicyId": 2,
            "virtualGuestMemberTemplate": {
                "accountId": 31111,
                "domain": "techsupport.com",
                "hostname": "ajcb-autoscale22",
                "maxMemory": 1024,
                "postInstallScriptUri": "https://pastebin.com/raw/62wrEKuW",
                "startCpus": 1,
                "blockDevices": [
                    {
                        "device": "0",
                        "diskImage": {
                            "capacity": 25,
                        }
                    },
                    {
                        "device": "2",
                        "diskImage": {
                            "capacity": 10,
                        }
                    }
                ],
                "datacenter": {
                    "name": "sao01",
                },
                "networkComponents": [
                    {
                        "maxSpeed": 100,
                    }
                ],
                "operatingSystemReferenceCode": "CENTOS_LATEST",
                "sshKeys": [
                    {
                        "id": 49111,
                    }
                ]
            },
            "logs": [
                {
                    "createDate": "2019-09-28T02:31:35+08:00",
                    "description": "Scaling group to 3 member(s) by removing -1 member(s) as manually requested",
                    "id": 3821111,
                    "scaleGroupId": 2251111,
                },
                {
                    "createDate": "2019-09-28T02:26:11+08:00",
                    "description": "Scaling group to 4 member(s) by adding 2 member(s) as manually requested",
                    "id": 38211111,
                    "scaleGroupId": 2251111,
                },
            ]
        }
    },
]
<commit_msg>Update SoftLayer_Scale_Group.py<commit_after>getObject = {
    "accountId": 31111,
    "cooldown": 1800,
    "createDate": "2016-10-25T01:48:34+08:00",
    "id": 12222222,
    "lastActionDate": "2016-10-25T01:48:34+08:00",
    "maximumMemberCount": 5,
    "minimumMemberCount": 0,
    "name": "tests",
    "regionalGroupId": 663,
    "virtualGuestMemberTemplate": {
        "accountId": 31111,
        "domain": "sodg.com",
        "hostname": "testing",
        "id": None,
        "maxCpu": None,
        "maxMemory": 32768,
        "startCpus": 32,
        "blockDevices": [
            {
                "device": "0",
                "diskImage": {
                    "capacity": 25,
                }
            }
        ],
        "datacenter": {
            "name": "sao01",
        },
        "hourlyBillingFlag": True,
        "operatingSystemReferenceCode": "CENTOS_LATEST",
        "privateNetworkOnlyFlag": True
    },
    "virtualGuestMemberCount": 0,
    "status": {
        "id": 1,
        "keyName": "ACTIVE",
        "name": "Active"
    },
    "virtualGuestAssets": [],
    "virtualGuestMembers": []
}

scale = [
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2016-10-25T01:48:34+08:00",
        "id": 12222222,
        "maximumMemberCount": 5,
        "minimumMemberCount": 0,
        "name": "tests",
        "virtualGuest": {
            "accountId": 31111,
            "createDate": "2019-10-02T15:24:54-06:00",
            "billingItem": {
                "cancellationDate": "2019-10-02T08:34:21-06:00"}
        },
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "sodg.com",
            "hostname": "testing",
            "maxMemory": 32768,
            "startCpus": 32,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "sao01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_LATEST",
            "privateNetworkOnlyFlag": True
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2018-04-24T04:22:00+08:00",
        "id": 224533333,
        "maximumMemberCount": 10,
        "minimumMemberCount": 0,
        "modifyDate": "2019-01-19T04:53:21+08:00",
        "name": "test-ajcb",
        "virtualGuest": {
            "accountId": 31111,
            "createDate": "2019-10-02T15:24:54-06:00",
            "billingItem": {
                "cancellationDate": "2019-10-02T08:34:21-06:00"}
        },
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "test.local",
            "hostname": "autoscale-ajcb01",
            "id": None,
            "maxCpu": None,
            "maxMemory": 1024,
            "postInstallScriptUri": "http://test.com",
            "startCpus": 1,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "seo01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_7_64",
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    }
]

scaleTo = [
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2016-10-25T01:48:34+08:00",
        "id": 12222222,
        "lastActionDate": "2016-10-25T01:48:34+08:00",
        "maximumMemberCount": 5,
        "minimumMemberCount": 0,
        "name": "tests",
        "regionalGroupId": 663,
        "virtualGuest": {
        },
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "sodg.com",
            "hostname": "testing",
            "id": None,
            "maxCpu": None,
            "maxMemory": 32768,
            "startCpus": 32,
            "datacenter": {
                "name": "sao01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_LATEST",
            "privateNetworkOnlyFlag": True
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
    {
        "accountId": 31111,
        "cooldown": 1800,
        "createDate": "2018-04-24T04:22:00+08:00",
        "id": 224533333,
        "lastActionDate": "2019-01-19T04:53:18+08:00",
        "maximumMemberCount": 10,
        "minimumMemberCount": 0,
        "modifyDate": "2019-01-19T04:53:21+08:00",
        "name": "test-ajcb",
        "regionalGroupId": 1025,
        "virtualGuest": {
            "accountId": 31111,
            "createDate": "2019-10-02T15:24:54-06:00",
            "billingItem": {
                "cancellationDate": "2019-10-02T08:34:21-06:00"}
        },
        "virtualGuestMemberTemplate": {
            "accountId": 31111,
            "domain": "test.local",
            "hostname": "autoscale-ajcb01",
            "id": None,
            "maxCpu": None,
            "maxMemory": 1024,
            "postInstallScriptUri": "http://test.com",
            "startCpus": 1,
            "blockDevices": [
                {
                    "device": "0",
                    "diskImage": {
                        "capacity": 25,
                    }
                }
            ],
            "datacenter": {
                "name": "seo01",
            },
            "hourlyBillingFlag": True,
            "operatingSystemReferenceCode": "CENTOS_7_64",
        },
        "virtualGuestMemberCount": 0,
        "status": {
            "id": 1,
            "keyName": "ACTIVE",
            "name": "Active"
        },
        "virtualGuestAssets": [],
        "virtualGuestMembers": []
    },
]

getLogs = [
    {
        "createDate": "2019-10-03T04:26:11+08:00",
        "description": "Scaling group to 6 member(s) by adding 3 member(s) as manually requested",
        "id": 3821111,
        "scaleGroupId": 2252222,
        "scaleGroup": {
            "accountId": 31111,
            "cooldown": 1800,
            "createDate": "2018-05-01T03:07:40+08:00",
            "id": 2251111,
            "lastActionDate": "2019-10-03T04:26:17+08:00",
            "maximumMemberCount": 6,
            "minimumMemberCount": 2,
            "modifyDate": "2019-10-03T04:26:21+08:00",
            "name": "ajcb-autoscale11",
            "regionalGroupId": 663,
            "terminationPolicyId": 2,
            "virtualGuestMemberTemplate": {
                "accountId": 31111,
                "domain": "techsupport.com",
                "hostname": "ajcb-autoscale22",
                "maxMemory": 1024,
                "postInstallScriptUri": "https://pastebin.com/raw/62wrEKuW",
                "startCpus": 1,
                "blockDevices": [
                    {
                        "device": "0",
                        "diskImage": {
                            "capacity": 25,
                        }
                    },
                    {
                        "device": "2",
                        "diskImage": {
                            "capacity": 10,
                        }
                    }
                ],
                "datacenter": {
                    "name": "sao01",
                },
                "networkComponents": [
                    {
                        "maxSpeed": 100,
                    }
                ],
                "operatingSystemReferenceCode": "CENTOS_LATEST",
                "sshKeys": [
                    {
                        "id": 49111,
                    }
                ]
            },
            "logs": [
                {
                    "createDate": "2019-09-28T02:31:35+08:00",
                    "description": "Scaling group to 3 member(s) by removing -1 member(s) as manually requested",
                    "id": 3821111,
                    "scaleGroupId": 2251111,
                },
                {
                    "createDate": "2019-09-28T02:26:11+08:00",
                    "description": "Scaling group to 4 member(s) by adding 2 member(s) as manually requested",
                    "id": 38211111,
                    "scaleGroupId": 2251111,
                },
            ]
        }
    },
]
<|endoftext|>


<commit_before>from .voc import VOCDetection, AnnotationTransform, detection_collate, VOC_CLASSES
from .data_augmentor import *
from .config import *
<commit_msg>rename data augmentation<commit_after>from .voc import VOCDetection, AnnotationTransform, detection_collate, VOC_CLASSES
from .data_augment import *
from .config import *
<|endoftext|>