<commit_before>import logging
import os
import sys
import unittest

testfile_path = os.path.realpath(os.path.abspath(__file__))
sys.path.append(os.path.dirname(os.path.dirname(testfile_path)))

from pycis import items
from pycis import wrappers

class ItemsTestCase(unittest.TestCase):

    def test_media_items_instantiantion(self):
        title = "The Animal Kingdom"
        url = "https://archive.org/details/Animal_Kingdom"
        media = items.Media(title=title, url=url)

        self.assertEqual(media.title, title)
        self.assertEqual(media.url, url)
        self.assertNotEqual(media.title, "wrong title")

    def test_media_str_representation(self):
        title = "The Animal Kingdom"
        url = "https://archive.org/details/Animal_Kingdom"
        media = items.Media(title=title, url=url)
        media_with_children = items.Media(title=title, url=url, has_children=True)

        expected_str = 'Media(title="The Animal Kingdom", url="https://archive.org/details/Animal_Kingdom", has_children={})'
        self.assertEqual(str(media), expected_str.format(False))
        self.assertEqual(str(media_with_children), expected_str.format(True))

class WrappersTestCase(unittest.TestCase):

    def test_get_wrapper_list_return_list(self):
        wrapper_list = wrappers.get_wrapper_list()
        self.assertIsInstance(wrapper_list, list)

    def test_get_wrapper_list_return_list_doesnt_include_base_wrapper(self):
        from pycis.wrappers.base_wrapper import BaseWrapper

        wrapper_list = wrappers.get_wrapper_list()
        for wrapper in wrapper_list:
            self.assertNotEqual(wrapper.name, "base")

        self.assertEqual(wrappers.get_wrapper(name="base"), None)

    def test_get_wrapper_by_name_is_callable(self):
        self.assertTrue(callable(wrappers.get_wrapper))

class ArchiveWrapperTest(unittest.TestCase):

    def test_wrapper_search_return_media_list(self):
        from pycis.wrappers.base_wrapper import BaseWrapper
        archive_wrapper = wrappers.get_wrapper("archive")

        self.assertIsInstance(archive_wrapper, BaseWrapper)
        media_list = archive_wrapper.search("the animal kingdom")

        self.assertIsInstance(media_list, list)

        for media in media_list:
            self.assertIsInstance(media, items.Media)

class TubePlusWrapperTest(unittest.TestCase):

    def setUp(self):
        from pycis.wrappers.base_wrapper import BaseWrapper
        self.tubeplus_wrapper = wrappers.get_wrapper("tubeplus")

    def test_tubeplus_wrapper_search_returns_right_media(self):
        media_list = self.tubeplus_wrapper.search("Eat Pray Love")

        self.assertIsInstance(media_list, list)
        self.assertIn("Eat Pray Love", (m.title for m in media_list))

    def test_tubeplus_wrapper_get_streams_return_list(self):
        media_list = self.tubeplus_wrapper.search_film("Eat Pray Love")
        first_media = media_list[0]
        stream_list = self.tubeplus_wrapper.get_streams(first_media)
        self.assertIsInstance(stream_list, list)

    def test_tubeplus_wrapper_get_children(self):
        media_list = self.tubeplus_wrapper.search_tvshow("Vampire Diaries")
        first_media = media_list[0]

        children_list = self.tubeplus_wrapper.get_children(first_media)

        self.assertIsInstance(children_list, list)
        self.assertTrue(all(m for m in children_list if isinstance(m, items.Media)))

if __name__ == "__main__":
    unittest.main()
<commit_msg>Modified tests to be only one test case for package<commit_after>import logging
import os
import sys
import unittest

testfile_path = os.path.realpath(os.path.abspath(__file__))
sys.path.append(os.path.dirname(os.path.dirname(testfile_path)))

from pycis import items
from pycis import wrappers
from pycis.wrappers.base_wrapper import BaseWrapper

class ItemsTestCase(unittest.TestCase):

    def test_media_items_instantiantion(self):
        title = "The Animal Kingdom"
        url = "https://archive.org/details/Animal_Kingdom"
        media = items.Media(title=title, url=url)

        self.assertEqual(media.title, title)
        self.assertEqual(media.url, url)
        self.assertNotEqual(media.title, "wrong title")

    def test_media_str_representation(self):
        title = "The Animal Kingdom"
        url = "https://archive.org/details/Animal_Kingdom"
        media = items.Media(title=title, url=url)
        media_with_children = items.Media(title=title, url=url, has_children=True)

        expected_str = 'Media(title="The Animal Kingdom", url="https://archive.org/details/Animal_Kingdom", has_children={})'
        self.assertEqual(str(media), expected_str.format(False))
        self.assertEqual(str(media_with_children), expected_str.format(True))

class WrappersTestCase(unittest.TestCase):

    def setUp(self):
        self.tubeplus_wrapper = wrappers.get_wrapper("tubeplus")
        self.archive_wrapper = wrappers.get_wrapper("archive")

    def test_get_wrapper_list_return_list(self):
        wrapper_list = wrappers.get_wrapper_list()
        self.assertIsInstance(wrapper_list, list)

    def test_get_wrapper_list_return_list_doesnt_include_base_wrapper(self):
        from pycis.wrappers.base_wrapper import BaseWrapper

        wrapper_list = wrappers.get_wrapper_list()
        for wrapper in wrapper_list:
            self.assertNotEqual(wrapper.name, "base")

        self.assertEqual(wrappers.get_wrapper(name="base"), None)

    def test_get_wrapper_by_name_is_callable(self):
        self.assertTrue(callable(wrappers.get_wrapper))

    def test_archive_wrapper_search_return_media_list(self):
        self.assertIsInstance(self.archive_wrapper, BaseWrapper)
        media_list = self.archive_wrapper.search("the animal kingdom")

        self.assertIsInstance(media_list, list)

        for media in media_list:
            self.assertIsInstance(media, items.Media)

    def test_tubeplus_wrapper_search_returns_right_media(self):
        media_list = self.tubeplus_wrapper.search("Eat Pray Love")

        self.assertIsInstance(media_list, list)
        self.assertIn("Eat Pray Love", (m.title for m in media_list))

    def test_tubeplus_wrapper_get_streams_return_list(self):
        media_list = self.tubeplus_wrapper.search_film("Eat Pray Love")
        first_media = media_list[0]
        stream_list = self.tubeplus_wrapper.get_streams(first_media)
        self.assertIsInstance(stream_list, list)

    def test_tubeplus_wrapper_get_children(self):
        media_list = self.tubeplus_wrapper.search_tvshow("Vampire Diaries")
        first_media = media_list[0]

        children_list = self.tubeplus_wrapper.get_children(first_media)

        self.assertIsInstance(children_list, list)
        self.assertTrue(all(m for m in children_list if isinstance(m, items.Media)))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>"""Make writing tests easier."""
import asyncio
import functools
import inspect

import aiohttp
import yarl

def _run_in_loop(coro):

    @functools.wraps(coro)
    def inner(self, *args, **kwargs):
        return self.loop.run_until_complete(coro(self, *args, **kwargs))

    return inner

def _init_loop(func):

    @functools.wraps(func)
    def inner(self):
        val = func(self)

        asyncio.set_event_loop(None)
        self.loop = asyncio.new_event_loop()
        return val
    return inner

def _close_loop(func):

    @functools.wraps(func)
    def inner(self):
        val = func(self)

        if not self.loop.is_closed():
            self.loop.call_soon(self.loop.stop)
            self.loop.run_forever()  # FIXME Is this needed?
            self.loop.close()
        return val
    return inner

class AsyncMeta(type):
    """Transform any class member coroutine function to a regular one."""

    def __new__(cls, name, bases, namespace, **kwds):
        """
        Create a modified cls object.

        Decorate `setUp` and `tearDown` methods to open and close an event loop
        properly and decorate all coroutines to make them run in an event loop
        using `asyncio.AbstractEventLoop.run_until_complete` function.
        """
        result = super().__new__(cls, name, bases, namespace, **kwds)
        for name, coro in inspect.getmembers(result, predicate=inspect.iscoroutinefunction):
            setattr(result, name, _run_in_loop(coro))

        if hasattr(result, 'setUp'):
            setattr(result, 'setUp', _init_loop(result.setUp))

        if hasattr(result, 'tearDown'):
            setattr(result, 'tearDown', _init_loop(result.tearDown))

        return result

class _FakeClientResponse(aiohttp.ClientResponse):
    def __init__(self, method, url, content, *args):
        if isinstance(url, str):
            url = yarl.URL(url)
        self._fake_content = content
        super().__init__(method, url, *args)

    async def read(self):
        return self._fake_content.encode('utf8')

    async def text(self):
        return self._fake_content

def fake_response(content, method='', *args):
    """Create a `aiohttp.ClientResponse` instance with fake content."""
    return _FakeClientResponse(method, yarl.URL(''), content, *args)
<commit_msg>[bug] Actually _close_ event loop in tests<commit_after>"""Make writing tests easier."""
import asyncio
import functools
import inspect

import aiohttp
import yarl

def _run_in_loop(coro):

    @functools.wraps(coro)
    def inner(self, *args, **kwargs):
        return self.loop.run_until_complete(coro(self, *args, **kwargs))

    return inner

def _init_loop(func):

    @functools.wraps(func)
    def inner(self):
        val = func(self)

        asyncio.set_event_loop(None)
        self.loop = asyncio.new_event_loop()
        return val
    return inner

def _close_loop(func):

    @functools.wraps(func)
    def inner(self):
        val = func(self)

        if not self.loop.is_closed():
            self.loop.call_soon(self.loop.stop)
            self.loop.run_forever()  # FIXME Is this needed?
            self.loop.close()
        return val
    return inner

class AsyncMeta(type):
    """Transform any class member coroutine function to a regular one."""

    def __new__(cls, name, bases, namespace, **kwds):
        """
        Create a modified cls object.

        Decorate `setUp` and `tearDown` methods to open and close an event loop
        properly and decorate all coroutines to make them run in an event loop
        using `asyncio.AbstractEventLoop.run_until_complete` function.
        """
        result = super().__new__(cls, name, bases, namespace, **kwds)
        for name, coro in inspect.getmembers(result, predicate=inspect.iscoroutinefunction):
            setattr(result, name, _run_in_loop(coro))

        if hasattr(result, 'setUp'):
            setattr(result, 'setUp', _init_loop(result.setUp))
        else:
            setattr(result, 'setUp', _init_loop(lambda self: None))

        if hasattr(result, 'tearDown'):
            setattr(result, 'tearDown', _close_loop(result.tearDown))
        else:
            setattr(result, 'tearDown', _close_loop(lambda self: None))

        return result

class _FakeClientResponse(aiohttp.ClientResponse):
    def __init__(self, method, url, content, *args):
        if isinstance(url, str):
            url = yarl.URL(url)
        self._fake_content = content
        super().__init__(method, url, *args)

    async def read(self):
        return self._fake_content.encode('utf8')

    async def text(self):
        return self._fake_content

def fake_response(content, method='', *args):
    """Create a `aiohttp.ClientResponse` instance with fake content."""
    return _FakeClientResponse(method, yarl.URL(''), content, *args)
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import *  # NOQA
from future import standard_library
standard_library.install_aliases()

import copy
from future.utils import native
from logging import getLogger

import chainer
from chainer import cuda
import chainer.functions as F
import numpy as np

from chainerrl import agent
from chainerrl.misc.copy_param import synchronize_parameters
from chainerrl.recurrent import Recurrent
from chainerrl.recurrent import state_reset
from chainerrl.replay_buffer import batch_experiences
from chainerrl.replay_buffer import batch_states
from chainerrl.replay_buffer import ReplayUpdator

def _to_device(obj, gpu):
    if isinstance(obj, tuple):
        return tuple(_to_device(x, gpu) for x in obj)
    else:
        if gpu >= 0:
            return cuda.to_gpu(obj, gpu)
        else:
            return cuda.to_cpu(obj)

def compute_value_loss(y, t, clip_delta=True, batch_accumulator='mean'):
    """Compute a loss for value prediction problem.

    Args:
        y (Variable or ndarray): Predicted values.
        t (Variable or ndarray): Target values.
        clip_delta (bool): Use the Huber loss function if set True.
        batch_accumulator (str): 'mean' or 'sum'. 'mean' will use the mean of
            the loss values in a batch. 'sum' will use the sum.
    Returns:
        (Variable) scalar loss
    """
    assert batch_accumulator in ('mean', 'sum')
    y = F.reshape(y, (-1, 1))
    t = F.reshape(t, (-1, 1))
    if clip_delta:
        loss_sum = F.sum(F.huber_loss(y, t, delta=1.0))
        if batch_accumulator == 'mean':
            loss = loss_sum / y.shape[0]
        elif batch_accumulator == 'sum':
            loss = loss_sum
    else:
        loss_mean = F.mean_squared_error(y, t) / 2
        if batch_accumulator == 'mean':
            loss = loss_mean
        elif batch_accumulator == 'sum':
            loss = loss_mean * y.shape[0]
    return loss

class DQN(agent.AttributeSavingMixin, agent.Agent):
    """Deep Q-Network algorithm.

    Args:
        q_function (StateQFunction): Q-function
        optimizer (Optimizer): Optimizer that is already setup
        replay_buffer (ReplayBuffer): Replay buffer
        gamma (float): Discount factor
        explorer (Explorer): Explorer that specifies an exploration strategy.
        gpu (int): GPU device id. -1 for CPU.
        replay_start_size (int): if the replay buffer's size is less than
            replay_start_size, skip update
        minibatch_size (int): Minibatch size
        update_frequency (int): Model update frequency in step
        target_update_frequency (int): Target model update frequency in step
        clip_delta (bool): Clip delta if set True
        phi (callable): Feature extractor applied to observations
        target_update_method (str): 'hard' or 'soft'.
        soft_update_tau (float): Tau of soft target update.
        n_times_update (int): Number of repetition of update
        average_q_decay (float): Decay rate of average Q, only used for
            recording statistics
        average_loss_decay (float): Decay rate of average loss, only used for
            recording statistics
        batch_accumulator (str): 'mean' or 'sum'
        episodic_update (bool): Use full episodes for update if set True
        episodic_update_len (int or None): Subsequences of this length are used
            for update if set int and episodic_update=True
        logger (Logger): Logger used
    """

    saved_attributes = ('model', 'target_model', 'optimizer')

    def __init__(self, q_function, optimizer, replay_buffer, gamma,
                 explorer, gpu=-1, replay_start_size=50000,
                 minibatch_size=32, update_frequency=1,
                 target_update_frequency=10000, clip_delta=True,
                 clip_reward=True, phi=lambda x: x,
                 target_update_method='hard',
                 soft_update_tau=1e-2,
                 n_times_update=1, average_q_decay=0.999,
                 average_loss_decay=0.99,
                 batch_accumulator='mean', episodic_update=False,
                 episodic_update_len=None,
                 logger=getLogger(__name__),
                 batch_states=batch_states):
        self.model = q_function
        self.q_function = q_function  # For backward compatibility

        # Future's builtins.int is a new type that inherit long, but Chainer
        # 1.15 only accepts int and long, so here we should use a native type.
        gpu = native(gpu)

        if gpu >= 0:
            cuda.get_device(gpu).use()
            self.model.to_gpu(device=gpu)
            self.xp = cuda.cupy
        else:
            self.xp = np
        self.replay_buffer = replay_buffer
        self.optimizer = optimizer
        self.gamma = gamma
        self.explorer = explorer
        self.gpu = gpu
        self.target_update_frequency = target_update_frequency
        self.clip_delta = clip_delta
        self.clip_reward = clip_reward
        self.phi = phi
        self.target_update_method = target_update_method
        self.soft_update_tau = soft_update_tau
        self.batch_accumulator = batch_accumulator
        assert batch_accumulator in ('mean', 'sum')
        self.logger = logger
        self.batch_states = batch_states
        if episodic_update:
            update_func = self.update_from_episodes
        else:
            update_func = self.update
        self.replay_updator = ReplayUpdator(
            replay_buffer=replay_buffer,
            update_func=update_func,
            batchsize=minibatch_size,
            episodic_update=episodic_update,
            episodic_update_len=episodic_update_len,
            n_times_update=n_times_update,
            replay_start_size=replay_start_size,
            update_frequency=update_frequency,
        )

        self.t = 0
        self.last_state = None
        self.last_action = None
        self.target_model = None
        self.sync_target_network()
        # For backward compatibility
        self.target_q_function = self.target_model
        self.average_q = 0
        self.average_q_decay = average_q_decay
        self.average_loss = 0
        self.average_loss_decay = average_loss_decay

    def sync_target_network(self):
        """Synchronize target network with current network."""
        if self.target_model is None:
            self.target_model = copy.deepcopy(self.model)
        else:
            synchronize_parameters(
                src=self.model,
                dst=self.target_model,
                method=self.target_update_method,
                tau=self.soft_update_tau)

    def update(self, experiences, errors_out=None):
        """Update the model from experiences

        This function is thread-safe.
        Args:
          experiences (list): list of dict that contains
            state: cupy.ndarray or numpy.ndarray
            action: int [0, n_action_types)
            reward: float32
            next_state: cupy.ndarray or numpy.ndarray
            next_legal_actions: list of booleans; True means legal
          gamma (float): discount factor
        Returns:
          None
        """

        exp_batch = batch_experiences(experiences, xp=self.xp, phi=self.phi,
                                      batch_states=self.batch_states)
        loss = self._compute_loss(
            exp_batch, self.gamma, errors_out=errors_out)

        # Update stats
        self.average_loss *= self.average_loss_decay
        self.average_loss += (1 - self.average_loss_decay) * float(loss.data)

        self.optimizer.zero_grads()
        loss.backward()
        self.optimizer.update()

    def input_initial_batch_to_target_model(self, batch):
        self.target_model(batch['state'])

    def update_from_episodes(self, episodes, errors_out=None):
        with state_reset(self.model):
            with state_reset(self.target_model):
                loss = 0
                sorted_episodes = list(reversed(sorted(episodes, key=len)))
                max_epi_len = len(sorted_episodes[0])
                for i in range(max_epi_len):
                    transitions = []
                    for ep in sorted_episodes:
                        if len(ep) <= i:
                            break
                        transitions.append(ep[i])
                    batch = batch_experiences(transitions,
                                              xp=self.xp,
                                              phi=self.phi,
                                              batch_states=self.batch_states)
                    if i == 0:
                        self.input_initial_batch_to_target_model(batch)
                    loss += self._compute_loss(batch, self.gamma)
                loss /= max_epi_len
                self.optimizer.zero_grads()
                loss.backward()
                self.optimizer.update()

    def _compute_target_values(self, exp_batch, gamma):

        batch_next_state = exp_batch['next_state']

        target_next_qout = self.target_model(batch_next_state, test=True)
        next_q_max = target_next_qout.max

        batch_rewards = exp_batch['reward']
        batch_terminal = exp_batch['is_state_terminal']

        return batch_rewards + self.gamma * (1.0 - batch_terminal) * next_q_max

    def _compute_y_and_t(self, exp_batch, gamma):
        batch_size = exp_batch['reward'].shape[0]

        # Compute Q-values for current states
        batch_state = exp_batch['state']

        qout = self.model(batch_state, test=False)

        batch_actions = exp_batch['action']
        batch_q = F.reshape(qout.evaluate_actions(
            batch_actions), (batch_size, 1))

        with chainer.no_backprop_mode():
            batch_q_target = F.reshape(
                self._compute_target_values(exp_batch, gamma),
                (batch_size, 1))

        return batch_q, batch_q_target

    def _compute_loss(self, exp_batch, gamma, errors_out=None):
        """Compute the Q-learning loss for a batch of experiences

        Args:
          experiences (list): see update()'s docstring
          gamma (float): discount factor
        Returns:
          loss
        """

        y, t = self._compute_y_and_t(exp_batch, gamma)

        if errors_out is not None:
            del errors_out[:]
            delta = F.sum(F.basic_math.absolute(y - t), axis=1)
            delta = cuda.to_cpu(delta.data)
            for e in delta:
                errors_out.append(e)

        return compute_value_loss(y, t, clip_delta=self.clip_delta,
                                  batch_accumulator=self.batch_accumulator)

    def compute_q_values(self, states):
        """Compute Q-values

        Args:
          states (list of cupy.ndarray or numpy.ndarray)
        Returns:
          list of numpy.ndarray
        """
        if not states:
            return []
        batch_x = self.batch_states(states, self.xp, self.phi)
        q_values = list(cuda.to_cpu(
            self.model(batch_x, test=True).q_values))
        return q_values

    def _to_my_device(self, model):
        if self.gpu >= 0:
            model.to_gpu(self.gpu)
        else:
            model.to_cpu()

    def act(self, state):
        qout = self.model(self.batch_states([state], self.xp, self.phi),
                          test=True)
        action = cuda.to_cpu(qout.greedy_actions.data)[0]
        action_var = chainer.Variable(self.xp.asarray([action]))
        q = float(qout.evaluate_actions(action_var).data)

        # Update stats
        self.average_q *= self.average_q_decay
        self.average_q += (1 - self.average_q_decay) * q

        self.logger.debug('t:%s a:%s q:%s qout:%s', self.t, action, q, qout)
        return action

    def act_and_train(self, state, reward):

        self.logger.debug('t:%s r:%s', self.t, reward)

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        greedy_action = self.act(state)
        action = self.explorer.select_action(self.t, lambda: greedy_action)
        self.t += 1

        # Update the target network
        if self.t % self.target_update_frequency == 0:
            self.sync_target_network()

        if self.last_state is not None:
            assert self.last_action is not None
            # Add a transition to the replay buffer
            self.replay_buffer.append(
                state=self.last_state,
                action=self.last_action,
                reward=reward,
                next_state=state,
                next_action=action,
                is_state_terminal=False)

        self.last_state = state
        self.last_action = action

        self.replay_updator.update_if_necessary(self.t)

        return self.last_action

    def stop_episode_and_train(self, state, reward, done=False):
        """Observe a terminal state and a reward.

        This function must be called once when an episode terminates.
        """

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        assert self.last_state is not None
        assert self.last_action is not None

        # Add a transition to the replay buffer
        self.replay_buffer.append(
            state=self.last_state,
            action=self.last_action,
            reward=reward,
            next_state=state,
            next_action=self.last_action,
            is_state_terminal=done)

        self.stop_episode()

    def stop_episode(self):
        self.last_state = None
        self.last_action = None
        if isinstance(self.model, Recurrent):
            self.model.reset_state()
        self.replay_buffer.stop_current_episode()

    def get_statistics(self):
        return [
            ('average_q', self.average_q),
            ('average_loss', self.average_loss),
        ]
<commit_msg>Enable model.to_gpu before passing it to DQN<commit_after>from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import *  # NOQA
from future import standard_library
standard_library.install_aliases()

import copy
from future.utils import native
from logging import getLogger

import chainer
from chainer import cuda
import chainer.functions as F
import numpy as np

from chainerrl import agent
from chainerrl.misc.copy_param import synchronize_parameters
from chainerrl.recurrent import Recurrent
from chainerrl.recurrent import state_reset
from chainerrl.replay_buffer import batch_experiences
from chainerrl.replay_buffer import batch_states
from chainerrl.replay_buffer import ReplayUpdator

def _to_device(obj, gpu):
    if isinstance(obj, tuple):
        return tuple(_to_device(x, gpu) for x in obj)
    else:
        if gpu >= 0:
            return cuda.to_gpu(obj, gpu)
        else:
            return cuda.to_cpu(obj)

def compute_value_loss(y, t, clip_delta=True, batch_accumulator='mean'):
    """Compute a loss for value prediction problem.

    Args:
        y (Variable or ndarray): Predicted values.
        t (Variable or ndarray): Target values.
        clip_delta (bool): Use the Huber loss function if set True.
        batch_accumulator (str): 'mean' or 'sum'. 'mean' will use the mean of
            the loss values in a batch. 'sum' will use the sum.
    Returns:
        (Variable) scalar loss
    """
    assert batch_accumulator in ('mean', 'sum')
    y = F.reshape(y, (-1, 1))
    t = F.reshape(t, (-1, 1))
    if clip_delta:
        loss_sum = F.sum(F.huber_loss(y, t, delta=1.0))
        if batch_accumulator == 'mean':
            loss = loss_sum / y.shape[0]
        elif batch_accumulator == 'sum':
            loss = loss_sum
    else:
        loss_mean = F.mean_squared_error(y, t) / 2
        if batch_accumulator == 'mean':
            loss = loss_mean
        elif batch_accumulator == 'sum':
            loss = loss_mean * y.shape[0]
    return loss

class DQN(agent.AttributeSavingMixin, agent.Agent):
    """Deep Q-Network algorithm.

    Args:
        q_function (StateQFunction): Q-function
        optimizer (Optimizer): Optimizer that is already setup
        replay_buffer (ReplayBuffer): Replay buffer
        gamma (float): Discount factor
        explorer (Explorer): Explorer that specifies an exploration strategy.
        gpu (int): GPU device id if not None nor negative.
        replay_start_size (int): if the replay buffer's size is less than
            replay_start_size, skip update
        minibatch_size (int): Minibatch size
        update_frequency (int): Model update frequency in step
        target_update_frequency (int): Target model update frequency in step
        clip_delta (bool): Clip delta if set True
        phi (callable): Feature extractor applied to observations
        target_update_method (str): 'hard' or 'soft'.
        soft_update_tau (float): Tau of soft target update.
        n_times_update (int): Number of repetition of update
        average_q_decay (float): Decay rate of average Q, only used for
            recording statistics
        average_loss_decay (float): Decay rate of average loss, only used for
            recording statistics
        batch_accumulator (str): 'mean' or 'sum'
        episodic_update (bool): Use full episodes for update if set True
        episodic_update_len (int or None): Subsequences of this length are used
            for update if set int and episodic_update=True
        logger (Logger): Logger used
    """

    saved_attributes = ('model', 'target_model', 'optimizer')

    def __init__(self, q_function, optimizer, replay_buffer, gamma,
                 explorer, gpu=None, replay_start_size=50000,
                 minibatch_size=32, update_frequency=1,
                 target_update_frequency=10000, clip_delta=True,
                 clip_reward=True, phi=lambda x: x,
                 target_update_method='hard',
                 soft_update_tau=1e-2,
                 n_times_update=1, average_q_decay=0.999,
                 average_loss_decay=0.99,
                 batch_accumulator='mean', episodic_update=False,
                 episodic_update_len=None,
                 logger=getLogger(__name__),
                 batch_states=batch_states):
        self.model = q_function
        self.q_function = q_function  # For backward compatibility

        # Future's builtins.int is a new type that inherit long, but Chainer
        # 1.15 only accepts int and long, so here we should use a native type.
        gpu = native(gpu)

        if gpu is not None and gpu >= 0:
            cuda.get_device(gpu).use()
            self.model.to_gpu(device=gpu)

        self.xp = self.model.xp
        self.replay_buffer = replay_buffer
        self.optimizer = optimizer
        self.gamma = gamma
        self.explorer = explorer
        self.gpu = gpu
        self.target_update_frequency = target_update_frequency
        self.clip_delta = clip_delta
        self.clip_reward = clip_reward
        self.phi = phi
        self.target_update_method = target_update_method
        self.soft_update_tau = soft_update_tau
        self.batch_accumulator = batch_accumulator
        assert batch_accumulator in ('mean', 'sum')
        self.logger = logger
        self.batch_states = batch_states
        if episodic_update:
            update_func = self.update_from_episodes
        else:
            update_func = self.update
        self.replay_updator = ReplayUpdator(
            replay_buffer=replay_buffer,
            update_func=update_func,
            batchsize=minibatch_size,
            episodic_update=episodic_update,
            episodic_update_len=episodic_update_len,
            n_times_update=n_times_update,
            replay_start_size=replay_start_size,
            update_frequency=update_frequency,
        )

        self.t = 0
        self.last_state = None
        self.last_action = None
        self.target_model = None
        self.sync_target_network()
        # For backward compatibility
        self.target_q_function = self.target_model
        self.average_q = 0
        self.average_q_decay = average_q_decay
        self.average_loss = 0
        self.average_loss_decay = average_loss_decay

    def sync_target_network(self):
        """Synchronize target network with current network."""
        if self.target_model is None:
            self.target_model = copy.deepcopy(self.model)
        else:
            synchronize_parameters(
                src=self.model,
                dst=self.target_model,
                method=self.target_update_method,
                tau=self.soft_update_tau)

    def update(self, experiences, errors_out=None):
        """Update the model from experiences

        This function is thread-safe.
        Args:
          experiences (list): list of dict that contains
            state: cupy.ndarray or numpy.ndarray
            action: int [0, n_action_types)
            reward: float32
            next_state: cupy.ndarray or numpy.ndarray
            next_legal_actions: list of booleans; True means legal
          gamma (float): discount factor
        Returns:
          None
        """

        exp_batch = batch_experiences(experiences, xp=self.xp, phi=self.phi,
                                      batch_states=self.batch_states)
        loss = self._compute_loss(
            exp_batch, self.gamma, errors_out=errors_out)

        # Update stats
        self.average_loss *= self.average_loss_decay
        self.average_loss += (1 - self.average_loss_decay) * float(loss.data)

        self.optimizer.zero_grads()
        loss.backward()
        self.optimizer.update()

    def input_initial_batch_to_target_model(self, batch):
        self.target_model(batch['state'])

    def update_from_episodes(self, episodes, errors_out=None):
        with state_reset(self.model):
            with state_reset(self.target_model):
                loss = 0
                sorted_episodes = list(reversed(sorted(episodes, key=len)))
                max_epi_len = len(sorted_episodes[0])
                for i in range(max_epi_len):
                    transitions = []
                    for ep in sorted_episodes:
                        if len(ep) <= i:
                            break
                        transitions.append(ep[i])
                    batch = batch_experiences(transitions,
                                              xp=self.xp,
                                              phi=self.phi,
                                              batch_states=self.batch_states)
                    if i == 0:
                        self.input_initial_batch_to_target_model(batch)
                    loss += self._compute_loss(batch, self.gamma)
                loss /= max_epi_len
                self.optimizer.zero_grads()
                loss.backward()
                self.optimizer.update()

    def _compute_target_values(self, exp_batch, gamma):

        batch_next_state = exp_batch['next_state']

        target_next_qout = self.target_model(batch_next_state, test=True)
        next_q_max = target_next_qout.max

        batch_rewards = exp_batch['reward']
        batch_terminal = exp_batch['is_state_terminal']

        return batch_rewards + self.gamma * (1.0 - batch_terminal) * next_q_max

    def _compute_y_and_t(self, exp_batch, gamma):
        batch_size = exp_batch['reward'].shape[0]

        # Compute Q-values for current states
        batch_state = exp_batch['state']

        qout = self.model(batch_state, test=False)

        batch_actions = exp_batch['action']
        batch_q = F.reshape(qout.evaluate_actions(
            batch_actions), (batch_size, 1))

        with chainer.no_backprop_mode():
            batch_q_target = F.reshape(
                self._compute_target_values(exp_batch, gamma),
                (batch_size, 1))

        return batch_q, batch_q_target

    def _compute_loss(self, exp_batch, gamma, errors_out=None):
        """Compute the Q-learning loss for a batch of experiences

        Args:
          experiences (list): see update()'s docstring
          gamma (float): discount factor
        Returns:
          loss
        """

        y, t = self._compute_y_and_t(exp_batch, gamma)

        if errors_out is not None:
            del errors_out[:]
            delta = F.sum(F.basic_math.absolute(y - t), axis=1)
            delta = cuda.to_cpu(delta.data)
            for e in delta:
                errors_out.append(e)

        return compute_value_loss(y, t, clip_delta=self.clip_delta,
                                  batch_accumulator=self.batch_accumulator)

    def compute_q_values(self, states):
        """Compute Q-values

        Args:
          states (list of cupy.ndarray or numpy.ndarray)
        Returns:
          list of numpy.ndarray
        """
        if not states:
            return []
        batch_x = self.batch_states(states, self.xp, self.phi)
        q_values = list(cuda.to_cpu(
            self.model(batch_x, test=True).q_values))
        return q_values

    def _to_my_device(self, model):
        if self.gpu >= 0:
            model.to_gpu(self.gpu)
        else:
            model.to_cpu()

    def act(self, state):
        qout = self.model(self.batch_states([state], self.xp, self.phi),
                          test=True)
        action = cuda.to_cpu(qout.greedy_actions.data)[0]
        action_var = chainer.Variable(self.xp.asarray([action]))
        q = float(qout.evaluate_actions(action_var).data)

        # Update stats
        self.average_q *= self.average_q_decay
        self.average_q += (1 - self.average_q_decay) * q

        self.logger.debug('t:%s a:%s q:%s qout:%s', self.t, action, q, qout)
        return action

    def act_and_train(self, state, reward):

        self.logger.debug('t:%s r:%s', self.t, reward)

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        greedy_action = self.act(state)
        action = self.explorer.select_action(self.t, lambda: greedy_action)
        self.t += 1

        # Update the target network
        if self.t % self.target_update_frequency == 0:
            self.sync_target_network()

        if self.last_state is not None:
            assert self.last_action is not None
            # Add a transition to the replay buffer
            self.replay_buffer.append(
                state=self.last_state,
                action=self.last_action,
                reward=reward,
                next_state=state,
                next_action=action,
                is_state_terminal=False)

        self.last_state = state
        self.last_action = action

        self.replay_updator.update_if_necessary(self.t)

        return self.last_action

    def stop_episode_and_train(self, state, reward, done=False):
        """Observe a terminal state and a reward.

        This function must be called once when an episode terminates.
        """

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        assert self.last_state is not None
        assert self.last_action is not None

        # Add a transition to the replay buffer
        self.replay_buffer.append(
            state=self.last_state,
            action=self.last_action,
            reward=reward,
            next_state=state,
            next_action=self.last_action,
            is_state_terminal=done)

        self.stop_episode()

    def stop_episode(self):
        self.last_state = None
        self.last_action = None
        if isinstance(self.model, Recurrent):
            self.model.reset_state()
        self.replay_buffer.stop_current_episode()

    def get_statistics(self):
        return [
            ('average_q', self.average_q),
            ('average_loss', self.average_loss),
        ]
<|endoftext|>


<commit_before>from enum import Enum, unique

from django.contrib.auth.base_user import AbstractBaseUser
from django.db import models

class User(AbstractBaseUser):
    email = models.CharField(max_length=40, unique=True)
    name = models.CharField(max_length=255)
    mobile_number = models.CharField(max_length=255)

    @unique
    class Gender(Enum):
        MALE = 1
        FEMALE = 2
    gender = models.SmallIntegerField(choices=(
        (g.value, g.name.title())for g in Gender)
    )

    is_active = models.BooleanField()

    USERNAME_FIELD = 'email'
    REQUIRED_FIELDS = ['email', 'nane', 'mobile_number', 'gender']

    def get_full_name(self):
        return self.name

    def get_short_name(self):
        return self.name

<commit_msg>More fields for user model<commit_after>from enum import Enum, unique

from django.contrib.auth.base_user import AbstractBaseUser
from django.db import models

class User(AbstractBaseUser):
    email = models.CharField(max_length=40, unique=True)
    name = models.CharField(max_length=255)
    mobile_number = models.CharField(max_length=255, unique=True,
                                     help_text="ex: +6281xxxxxxx")
    whatsapp_number = models.CharField(max_length=255, unique=True,
                                       help_text="ex: +6281xxxxxx")
    pk_number = models.PositiveSmallIntegerField()
    previous_university = models.CharField(max_length=255, blank=True, null=True)

    @unique
    class Gender(Enum):
        MALE = 1
        FEMALE = 2
    gender = models.SmallIntegerField(choices=
        ((g.value, g.name.title()) for g in Gender)
    )

    @unique
    class Type(Enum):
        VOLUNTEER = 1
        INITIATOR = 2
    type = moderls.SmallIntegerField(choices=
        ((t.value, t.name.title()) for t in Type)
    )

    is_active = models.BooleanField(default=False)

    USERNAME_FIELD = 'email'
    REQUIRED_FIELDS = ['email', 'nane', 'mobile_number', 'gender']

    def get_full_name(self):
        return self.name

    def get_short_name(self):
        return self.name

    @property
    def first_name(self):
        return self.name.split(" ")[0]

<|endoftext|>


<commit_before>import asyncio
import asyncio.log
import datetime
import json
import re
import ssl
from collections import defaultdict
from email.utils import formatdate
from grp import getgrnam
from http import HTTPStatus
import logging
import os
import sys
from pwd import getpwnam
from typing import List
from urllib.parse import urlparse, parse_qs

from jsonget import json_get

from evtstrd.cmdargs import parse_command_line
from evtstrd.config import Config
from evtstrd.date import parse_iso_date
from evtstrd.events import JSONEvent, PingEvent
from evtstrd.exc import DisconnectedError
from evtstrd.http import \
    HTTPError, CGIArgumentError, NotFoundError, MethodNotAllowedError, \
    read_http_head, write_http_error, write_http_head, write_chunk, Header
from evtstrd.util import read_json_line

def run_notification_server():
    config = parse_command_line()
    asyncio.log.logger.disabled = True
    try:
        NotificationServer(config).run()
    except KeyboardInterrupt:
        pass

class NotificationServer:

    def __init__(self, config: Config) -> None:
        self._config = config
        self._loop = asyncio.get_event_loop()
        self._listeners = defaultdict(list)
        self._stats = ServerStats()
        self._socket_handler = SocketHandler(self._listeners, loop=self._loop)
        self._http_handler = HTTPHandler(
            config, self._listeners, self._stats, loop=self._loop)

    def run(self):
        self._remove_stale_socket()
        try:
            self._run_loop()
        finally:
            try:
                os.remove(self._config.socket_file)
            except FileNotFoundError:
                pass

    def _remove_stale_socket(self):
        if not os.path.exists(self._config.socket_file):
            return
        try:
            fut = asyncio.open_unix_connection(self._config.socket_file)
            self._loop.run_until_complete(fut)
        except ConnectionRefusedError:
            os.remove(self._config.socket_file)
            logging.warning("removed stale socket file {}".format(
                self._config.socket_file))
        else:
            print("server already running, exiting", file=sys.stderr)
            sys.exit(1)

    def _run_loop(self):
        self._start_socket()
        self._start_http_server()
        self._change_socket_permissions()
        self._loop.run_forever()

    def _start_socket(self):
        f = asyncio.start_unix_server(
            self._socket_handler.handle, path=self._config.socket_file)
        self._loop.run_until_complete(f)

    def _start_http_server(self):
        if self._config.with_ssl:
            ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
            ssl_context.load_cert_chain(
                self._config.cert_file, self._config.key_file)
        else:
            ssl_context = None
        f = asyncio.start_server(
            self._http_handler.handle, port=self._config.http_port,
            ssl=ssl_context)
        self._loop.run_until_complete(f)

    def _change_socket_permissions(self):
        os.chmod(self._config.socket_file, self._config.socket_mode)
        if self._config.socket_owner is None:
            new_owner = -1
        else:
            new_owner = getpwnam(self._config.socket_owner).pw_uid
        if self._config.socket_group is None:
            new_group = -1
        else:
            new_group = getgrnam(self._config.socket_group).gr_gid
        if new_owner != -1 or new_group != -1:
            os.chown(self._config.socket_file, new_owner, new_group)

class SocketHandler:

    def __init__(self, listeners, *, loop=None):
        self._listeners = listeners
        self._loop = loop or asyncio.get_event_loop()

    @asyncio.coroutine
    def handle(self, reader, writer):
        while True:
            try:
                message = yield from read_json_line(reader)
            except DisconnectedError:
                break
            action = json_get(message, "action", str)
            logging.debug("received a '{}' message".format(action))
            if action == "notify":
                self._notify_listeners_about_message(message)
            else:
                logging.warning("received unknown action '{}'".format(action))

    def _notify_listeners_about_message(self, message):
        try:
            subsystem, event, data, id = self._get_event_data(message)
        except ValueError:
            pass
        else:
            self._notify_listeners(subsystem, event, data, id)

    def _notify_listeners(self, subsystem, event_type, data, id):
        listeners = self._listeners[subsystem]
        # Copy the list of listeners, because it can be modified during the
        # iteration.
        for listener in listeners[:]:
            listener.notify(event_type, data, id)
        logging.info(
            "notified {} listeners about {} event in subsystem '{}'".format(
                len(listeners), event_type, subsystem))

    @staticmethod
    def _get_event_data(message):
        try:
            subsystem = json_get(message, "subsystem", str)
            event = json_get(message, "event", str)
            data = json_get(message, "data", dict)
            id = json_get(message, "id", str)
        except (ValueError, TypeError) as exc:
            logging.error("received invalid JSON: " + str(exc))
            raise ValueError()
        return subsystem, event, data, id

class HTTPHandler:

    def __init__(self, config: Config, listeners, stats, *, loop=None) -> None:
        self._config = config
        self._listeners = listeners
        self._stats = stats
        self._loop = loop or asyncio.get_event_loop()

    @asyncio.coroutine
    def handle(self, reader, writer):
        try:
            method, path, headers = yield from read_http_head(reader)
            yield from self._handle_request(
                reader, writer, method, path, headers)
        except HTTPError as exc:
            write_http_error(writer, exc)
        writer.close()

    @asyncio.coroutine
    def _handle_request(self, reader, writer, method, path, headers):
        url = urlparse(path)
        if url.path == "/events":
            if method != "GET":
                raise MethodNotAllowedError(method)
            yield from self._handle_get_events(reader, writer, url, headers)
        elif url.path == "/stats":
            if method != "GET":
                raise MethodNotAllowedError(method)
            self._handle_get_stats(writer)
        else:
            raise NotFoundError(path)

    def _default_headers(self) -> List[Header]:
        return [
            ("Date", formatdate(usegmt=True)),
            ("Server", "zzb-notificationd"),
        ]

    @asyncio.coroutine
    def _handle_get_events(self, reader, writer, url, headers):
        subsystem, filters = self._parse_event_args(url.query)
        response_headers = self._default_headers() + [
            ("Transfer-Encoding", "chunked"),
            ("Content-Type", "text/event-stream"),
            ("Connection", "keep-alive"),
            ("Keep-Alive", "timeout=5, max=100"),
        ]
        if "origin" in headers:
            response_headers.extend([
                ("Access-Control-Allow-Credentials", "true"),
                ("Access-Control-Allow-Origin", headers["origin"]),
            ])
        write_http_head(writer, HTTPStatus.OK, response_headers)
        yield from self._setup_listener(
            reader, writer, headers, subsystem, filters)

    @asyncio.coroutine
    def _setup_listener(self, reader, writer, headers, subsystem, filters):
        listener = self._create_listener(
            reader, writer, headers, subsystem, filters)
        self._listeners[subsystem].append(listener)
        self._stats.total_connections += 1
        yield from listener.ping_loop()

    def _create_listener(self, reader, writer, headers, subsystem, filters):
        logging.info("client subscribed to subsystem '{}'".format(subsystem))
        listener = Listener(
            self._config, reader, writer, subsystem, filters, loop=self._loop)
        listener.on_close = self._remove_listener
        listener.remote_host = writer.get_extra_info("peername")[0]
        listener.referer = headers.get("referer")
        return listener

    def _remove_listener(self, listener):
        logging.info(
            "client disconnected from subsystem '{}'".format(
                listener.subsystem))
        self._listeners[listener.subsystem].remove(listener)

    def _parse_event_args(self, query):
        args = parse_qs(query)
        if "subsystem" not in args:
            raise CGIArgumentError("subsystem", "missing argument")
        try:
            filters = [parse_filter(f) for f in args.get("filter", [])]
        except ValueError:
            raise CGIArgumentError("filter", "could not parse filter")
        return args["subsystem"][0], filters

    def _handle_get_stats(self, writer):
        all_listeners = []
        for key in self._listeners:
            all_listeners.extend(self._listeners[key])
        j = json_stats(self._stats, all_listeners)
        response = json.dumps(j).encode("utf-8")
        response_headers = self._default_headers() + [
            ("Connection", "close"),
            ("Content-Type", "application/json"),
            ("Content-Length", str(len(response)))
        ]
        write_http_head(writer, HTTPStatus.OK, response_headers)
        writer.write(response)
        writer.close()

class Listener:

    def __init__(
            self, config: Config, reader, writer, subsystem, filters,
            *, loop=None) -> None:
        self._config = config
        self.loop = loop or asyncio.get_event_loop()
        self.subsystem = subsystem
        self.filters = filters
        self.reader = reader
        self.writer = writer
        self.on_close = None
        self.connection_time = datetime.datetime.now()
        self.remote_host = None
        self.referer = None

    def __repr__(self):
        return "<Listener 0x{:x} for {}>".format(id(self), self.subsystem)

    def notify(self, event_type, data, id=None):
        if all(f(data) for f in self.filters):
            event = JSONEvent(event_type, data, id)
            try:
                self._write_event(event)
            except DisconnectedError:
                pass

    @asyncio.coroutine
    def ping_loop(self):
        while True:
            try:
                self._write_event(PingEvent())
            except DisconnectedError:
                break
            yield from asyncio.sleep(
                self._config.ping_interval, loop=self.loop)

    def _write_event(self, event):
        if self.reader.at_eof():
            if self.on_close:
                self.on_close(self)
            raise DisconnectedError()
        write_chunk(self.writer, bytes(event))

_filter_re = re.compile(r"^([a-z.-]+)(=|>=|<=)(.*)$")
_comparators = {
    "=": lambda v1, v2: v1 == v2,
    ">=": lambda v1, v2: v1 >= v2,
    "<=": lambda v1, v2: v1 <= v2,
}

def parse_filter(string):
    def parse_value(v):
        if len(v) >= 2 and v.startswith("'") and v.endswith("'"):
            return v[1:-1]
        try:
            return parse_iso_date(v)
        except ValueError:
            pass
        return int(v)

    m = _filter_re.match(string)
    if not m:
        raise ValueError()
    field = m.group(1).replace(".", "/")
    comparator = _comparators[m.group(2)]
    value = parse_value(m.group(3))
    if type(value) == datetime.date:
        cls = DateFilter
    else:
        cls = Filter
    filter_ = cls(field, comparator, value)
    filter_.string = string
    return filter_

class Filter:

    def __init__(self, field, comparator, value):
        self._field = field
        self._comparator = comparator
        self._value = value
        self.string = ""

    def __call__(self, message):
        try:
            v = self._get_value(message)
        except ValueError:
            return False
        return self._comparator(v, self._value)

    def __str__(self):
        return self.string

    def _get_value(self, message):
        try:
            v = json_get(message, self._field, self.field_type)
        except (ValueError, TypeError):
            raise ValueError()
        return self.parse_value(v)

    @property
    def field_type(self):
        return type(self._value)

    def parse_value(self, v):
        return v

class DateFilter(Filter):

    @property
    def field_type(self):
        return str

    def parse_value(self, v):
        return parse_iso_date(v)

class ServerStats:

    def __init__(self):
        self.start_time = datetime.datetime.now()
        self.total_connections = 0

def json_stats(stats, listeners):
    def json_connection(listener):
        c = {
            "subsystem": listener.subsystem,
            "filters": [str(f) for f in listener.filters],
            "connection-time": listener.connection_time.isoformat(),
            "remote-host": listener.remote_host,
        }
        if listener.referer:
            c["referer"] = listener.referer
        return c

    return {
        "start-time": stats.start_time.isoformat(),
        "total-connections": stats.total_connections,
        "connections": [
            json_connection(l) for l in listeners
        ],
    }
<commit_msg>shut down gracefully on SIGINT and SIGTERM<commit_after>import asyncio
import asyncio.log
import datetime
import json
import re
import signal
import ssl
from collections import defaultdict
from email.utils import formatdate
from grp import getgrnam
from http import HTTPStatus
import logging
import os
import sys
from pwd import getpwnam
from typing import List
from urllib.parse import urlparse, parse_qs

from jsonget import json_get

from evtstrd.cmdargs import parse_command_line
from evtstrd.config import Config
from evtstrd.date import parse_iso_date
from evtstrd.events import JSONEvent, PingEvent
from evtstrd.exc import DisconnectedError
from evtstrd.http import \
    HTTPError, CGIArgumentError, NotFoundError, MethodNotAllowedError, \
    read_http_head, write_http_error, write_http_head, write_chunk, Header
from evtstrd.util import read_json_line

def run_notification_server():
    config = parse_command_line()
    asyncio.log.logger.disabled = True
    NotificationServer(config).run()

class NotificationServer:

    def __init__(self, config: Config) -> None:
        self._config = config
        self._http_server = None
        self._socket_server = None
        self._loop = asyncio.get_event_loop()
        self._listeners = defaultdict(list)
        self._stats = ServerStats()
        self._socket_handler = SocketHandler(self._listeners, loop=self._loop)
        self._http_handler = HTTPHandler(
            config, self._listeners, self._stats, loop=self._loop)
        self._setup_signal_handlers()

    def _setup_signal_handlers(self):
        self._loop.add_signal_handler(signal.SIGINT, self._loop.stop)
        self._loop.add_signal_handler(signal.SIGTERM, self._loop.stop)

    def run(self):
        self._remove_stale_socket()
        try:
            self._run_loop()
        finally:
            try:
                os.remove(self._config.socket_file)
            except FileNotFoundError:
                pass

    def _remove_stale_socket(self):
        if not os.path.exists(self._config.socket_file):
            return
        try:
            fut = asyncio.open_unix_connection(self._config.socket_file)
            self._loop.run_until_complete(fut)
        except ConnectionRefusedError:
            os.remove(self._config.socket_file)
            logging.warning("removed stale socket file {}".format(
                self._config.socket_file))
        else:
            print("server already running, exiting", file=sys.stderr)
            sys.exit(1)

    def _run_loop(self):
        self._start_socket()
        self._start_http_server()
        self._change_socket_permissions()
        self._loop.run_forever()
        self._shutdown()

    def _shutdown(self):
        fs = []
        if self._http_server:
            self._http_server.close()
            fs.append(self._http_server.wait_closed())
        if self._socket_server:
            self._socket_server.close()
            fs.append(self._socket_server.wait_closed())
        self._loop.run_until_complete(asyncio.wait(fs, timeout=5))
        self._http_handler.disconnect_all()
        self._loop.shutdown_asyncgens()

    def _start_socket(self):
        f = asyncio.start_unix_server(
            self._socket_handler.handle, path=self._config.socket_file)
        self._socket_server = self._loop.run_until_complete(f)

    def _start_http_server(self):
        if self._config.with_ssl:
            ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
            ssl_context.load_cert_chain(
                self._config.cert_file, self._config.key_file)
        else:
            ssl_context = None
        f = asyncio.start_server(
            self._http_handler.handle, port=self._config.http_port,
            ssl=ssl_context)
        self._http_server = self._loop.run_until_complete(f)

    def _change_socket_permissions(self):
        os.chmod(self._config.socket_file, self._config.socket_mode)
        if self._config.socket_owner is None:
            new_owner = -1
        else:
            new_owner = getpwnam(self._config.socket_owner).pw_uid
        if self._config.socket_group is None:
            new_group = -1
        else:
            new_group = getgrnam(self._config.socket_group).gr_gid
        if new_owner != -1 or new_group != -1:
            os.chown(self._config.socket_file, new_owner, new_group)

class SocketHandler:

    def __init__(self, listeners, *, loop=None):
        self._listeners = listeners
        self._loop = loop or asyncio.get_event_loop()

    @asyncio.coroutine
    def handle(self, reader, writer):
        while True:
            try:
                message = yield from read_json_line(reader)
            except DisconnectedError:
                break
            action = json_get(message, "action", str)
            logging.debug("received a '{}' message".format(action))
            if action == "notify":
                self._notify_listeners_about_message(message)
            else:
                logging.warning("received unknown action '{}'".format(action))

    def _notify_listeners_about_message(self, message):
        try:
            subsystem, event, data, id = self._get_event_data(message)
        except ValueError:
            pass
        else:
            self._notify_listeners(subsystem, event, data, id)

    def _notify_listeners(self, subsystem, event_type, data, id):
        listeners = self._listeners[subsystem]
        # Copy the list of listeners, because it can be modified during the
        # iteration.
        for listener in listeners[:]:
            listener.notify(event_type, data, id)
        logging.info(
            "notified {} listeners about {} event in subsystem '{}'".format(
                len(listeners), event_type, subsystem))

    @staticmethod
    def _get_event_data(message):
        try:
            subsystem = json_get(message, "subsystem", str)
            event = json_get(message, "event", str)
            data = json_get(message, "data", dict)
            id = json_get(message, "id", str)
        except (ValueError, TypeError) as exc:
            logging.error("received invalid JSON: " + str(exc))
            raise ValueError()
        return subsystem, event, data, id

class HTTPHandler:

    def __init__(self, config: Config, listeners, stats, *, loop=None) -> None:
        self._config = config
        self._listeners = listeners
        self._stats = stats
        self._loop = loop or asyncio.get_event_loop()

    @asyncio.coroutine
    def handle(self, reader, writer):
        try:
            method, path, headers = yield from read_http_head(reader)
            yield from self._handle_request(
                reader, writer, method, path, headers)
        except HTTPError as exc:
            write_http_error(writer, exc)
        writer.close()

    @asyncio.coroutine
    def _handle_request(self, reader, writer, method, path, headers):
        url = urlparse(path)
        if url.path == "/events":
            if method != "GET":
                raise MethodNotAllowedError(method)
            yield from self._handle_get_events(reader, writer, url, headers)
        elif url.path == "/stats":
            if method != "GET":
                raise MethodNotAllowedError(method)
            self._handle_get_stats(writer)
        else:
            raise NotFoundError(path)

    def _default_headers(self) -> List[Header]:
        return [
            ("Date", formatdate(usegmt=True)),
            ("Server", "zzb-notificationd"),
        ]

    @asyncio.coroutine
    def _handle_get_events(self, reader, writer, url, headers):
        subsystem, filters = self._parse_event_args(url.query)
        response_headers = self._default_headers() + [
            ("Transfer-Encoding", "chunked"),
            ("Content-Type", "text/event-stream"),
            ("Connection", "keep-alive"),
            ("Keep-Alive", "timeout=5, max=100"),
        ]
        if "origin" in headers:
            response_headers.extend([
                ("Access-Control-Allow-Credentials", "true"),
                ("Access-Control-Allow-Origin", headers["origin"]),
            ])
        write_http_head(writer, HTTPStatus.OK, response_headers)
        yield from self._setup_listener(
            reader, writer, headers, subsystem, filters)

    @asyncio.coroutine
    def _setup_listener(self, reader, writer, headers, subsystem, filters):
        listener = self._create_listener(
            reader, writer, headers, subsystem, filters)
        self._listeners[subsystem].append(listener)
        self._stats.total_connections += 1
        yield from listener.ping_loop()

    def _create_listener(self, reader, writer, headers, subsystem, filters):
        logging.info("client subscribed to subsystem '{}'".format(subsystem))
        listener = Listener(
            self._config, reader, writer, subsystem, filters, loop=self._loop)
        listener.on_close = self._remove_listener
        listener.remote_host = writer.get_extra_info("peername")[0]
        listener.referer = headers.get("referer")
        return listener

    def _remove_listener(self, listener):
        logging.info(
            "client disconnected from subsystem '{}'".format(
                listener.subsystem))
        self._listeners[listener.subsystem].remove(listener)

    def _parse_event_args(self, query):
        args = parse_qs(query)
        if "subsystem" not in args:
            raise CGIArgumentError("subsystem", "missing argument")
        try:
            filters = [parse_filter(f) for f in args.get("filter", [])]
        except ValueError:
            raise CGIArgumentError("filter", "could not parse filter")
        return args["subsystem"][0], filters

    @property
    def _all_listeners(self):
        all_listeners = []
        for key in self._listeners:
            all_listeners.extend(self._listeners[key])
        return all_listeners

    def _handle_get_stats(self, writer):
        j = json_stats(self._stats,self._all_listeners)
        response = json.dumps(j).encode("utf-8")
        response_headers = self._default_headers() + [
            ("Connection", "close"),
            ("Content-Type", "application/json"),
            ("Content-Length", str(len(response)))
        ]
        write_http_head(writer, HTTPStatus.OK, response_headers)
        writer.write(response)
        writer.close()

    def disconnect_all(self):
        for listener in self._all_listeners:
            listener.disconnect()

class Listener:

    def __init__(
            self, config: Config, reader, writer, subsystem, filters,
            *, loop=None) -> None:
        self._config = config
        self.loop = loop or asyncio.get_event_loop()
        self.subsystem = subsystem
        self.filters = filters
        self.reader = reader
        self.writer = writer
        self.on_close = None
        self.connection_time = datetime.datetime.now()
        self.remote_host = None
        self.referer = None

    def __repr__(self):
        return "<Listener 0x{:x} for {}>".format(id(self), self.subsystem)

    def notify(self, event_type, data, id=None):
        if all(f(data) for f in self.filters):
            event = JSONEvent(event_type, data, id)
            try:
                self._write_event(event)
            except DisconnectedError:
                pass

    @asyncio.coroutine
    def ping_loop(self):
        while True:
            try:
                self._write_event(PingEvent())
            except DisconnectedError:
                break
            yield from asyncio.sleep(
                self._config.ping_interval, loop=self.loop)

    def _write_event(self, event):
        if self.reader.at_eof():
            if self.on_close:
                self.on_close(self)
            raise DisconnectedError()
        write_chunk(self.writer, bytes(event))

    def disconnect(self):
        self.writer.close()

_filter_re = re.compile(r"^([a-z.-]+)(=|>=|<=)(.*)$")
_comparators = {
    "=": lambda v1, v2: v1 == v2,
    ">=": lambda v1, v2: v1 >= v2,
    "<=": lambda v1, v2: v1 <= v2,
}

def parse_filter(string):
    def parse_value(v):
        if len(v) >= 2 and v.startswith("'") and v.endswith("'"):
            return v[1:-1]
        try:
            return parse_iso_date(v)
        except ValueError:
            pass
        return int(v)

    m = _filter_re.match(string)
    if not m:
        raise ValueError()
    field = m.group(1).replace(".", "/")
    comparator = _comparators[m.group(2)]
    value = parse_value(m.group(3))
    if type(value) == datetime.date:
        cls = DateFilter
    else:
        cls = Filter
    filter_ = cls(field, comparator, value)
    filter_.string = string
    return filter_

class Filter:

    def __init__(self, field, comparator, value):
        self._field = field
        self._comparator = comparator
        self._value = value
        self.string = ""

    def __call__(self, message):
        try:
            v = self._get_value(message)
        except ValueError:
            return False
        return self._comparator(v, self._value)

    def __str__(self):
        return self.string

    def _get_value(self, message):
        try:
            v = json_get(message, self._field, self.field_type)
        except (ValueError, TypeError):
            raise ValueError()
        return self.parse_value(v)

    @property
    def field_type(self):
        return type(self._value)

    def parse_value(self, v):
        return v

class DateFilter(Filter):

    @property
    def field_type(self):
        return str

    def parse_value(self, v):
        return parse_iso_date(v)

class ServerStats:

    def __init__(self):
        self.start_time = datetime.datetime.now()
        self.total_connections = 0

def json_stats(stats, listeners):
    def json_connection(listener):
        c = {
            "subsystem": listener.subsystem,
            "filters": [str(f) for f in listener.filters],
            "connection-time": listener.connection_time.isoformat(),
            "remote-host": listener.remote_host,
        }
        if listener.referer:
            c["referer"] = listener.referer
        return c

    return {
        "start-time": stats.start_time.isoformat(),
        "total-connections": stats.total_connections,
        "connections": [
            json_connection(l) for l in listeners
        ],
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import argparse
from functools import partial
from multiprocessing import Pool, cpu_count
from progressbar import ProgressBar
import time

from mapchete import *
from tilematrix import TilePyramid, MetaTilePyramid

def main(args):

    parser = argparse.ArgumentParser()
    parser.add_argument("mapchete_file", type=str)
    parser.add_argument("--zoom", "-z", type=int, nargs='*', )
    parser.add_argument("--bounds", "-b", type=float, nargs='*')
    parser.add_argument("--log", action="store_true")
    parser.add_argument("--overwrite", action="store_true")
    parsed = parser.parse_args(args)

    try:
        print "preparing process ..."
        process_host = MapcheteHost(
            parsed.mapchete_file,
            zoom=parsed.zoom,
            bounds=parsed.bounds
            )
    except Exception as e:
        raise

    work_tiles = process_host.get_work_tiles()

    print len(work_tiles), "tiles to be processed"

    overwrite = parsed.overwrite

    f = partial(worker,
        process_host=process_host,
        overwrite=overwrite
    )
    pool = Pool()
    logs = []

    try:
        output = pool.map_async(f, work_tiles, callback=logs.extend)
        total = output._number_left
        pbar = ProgressBar(maxval=total).start()
        pool.close()
        while (True):
            if (output.ready()): break
            counter = total - output._number_left
            pbar.update(counter)
            # update cycle set to one second
            time.sleep(1)
        pbar.finish()
    except KeyboardInterrupt:
        pool.terminate()
        sys.exit()
    except:
        raise
    finally:
        pool.close()
        pool.join()

    if process_host.config["output_format"] in [
        "GTiff",
        "PNG",
        "PNG_hillshade"
        ]:
        for zoom in process_host.config["zoom_levels"]:
            out_dir = os.path.join(
                process_host.config["output_name"],
                str(zoom)
            )
            out_vrt = os.path.join(
                process_host.config["output_name"],
                (str(zoom)+".vrt")
            )
            command = "gdalbuildvrt -overwrite %s %s" %(
                out_vrt,
                str(out_dir + "/*/*" + process_host.format.extension)
            )
            os.system(command)

    if parsed.log:
        for row in logs:
            if row[1] != "ok":
                print row

def worker(tile, process_host, overwrite):
    """
    Worker function running the process depending on the overwrite flag and
    whether the tile exists.
    """
    if not overwrite:
        image_path = process_host.tile_pyramid.format.get_tile_name(
            process_host.config["output_name"],
            tile
        )
        if os.path.isfile(image_path):
            print "exists"
            return tile, "exists", None
    log = process_host.save_tile(tile, overwrite)
    return log

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>add basic logging and catching traceback<commit_after>#!/usr/bin/env python

import os
import sys
import argparse
from functools import partial
from multiprocessing import Pool, cpu_count
import time
import logging
import logging.config
import traceback

from mapchete import *
from tilematrix import TilePyramid, MetaTilePyramid

logger = logging.getLogger("mapchete")

def main(args):

    parser = argparse.ArgumentParser()
    parser.add_argument("mapchete_file", type=str)
    parser.add_argument("--zoom", "-z", type=int, nargs='*', )
    parser.add_argument("--bounds", "-b", type=float, nargs='*')
    parser.add_argument("--log", action="store_true")
    parser.add_argument("--overwrite", action="store_true")
    parsed = parser.parse_args(args)

    log_dir = os.path.dirname(parsed.mapchete_file)
    log_file = os.path.join(log_dir, "mapchete.log")

    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': True,
        'formatters': {
            'simple': {
                'format': '%(levelname)s: %(message)s'
            },
            'verbose': {
                'format': '[%(asctime)s][%(module)s] %(levelname)s: %(message)s'
            }
        },
        'handlers': {
            'file': {
                'level': 'DEBUG',
                'class': 'logging.handlers.WatchedFileHandler',
                'filename': log_file,
                'formatter': 'verbose',
                'filters': [],
            },
            'stream': {
                'level': 'INFO',
                'class': 'logging.StreamHandler',
                'formatter': 'simple',
                'filters': [],
            }
        },
        'loggers': {
            'mapchete': {
                'handlers': ['file', 'stream'],
                'level': 'DEBUG',
                'propagate': True
            }
        }
    })

    try:
        logger.info("preparing process ...")
        process_host = MapcheteHost(
            parsed.mapchete_file,
            zoom=parsed.zoom,
            bounds=parsed.bounds
            )
    except Exception as e:
        raise

    work_tiles = process_host.get_work_tiles()

    logger.info("%d tiles to be processed" % len(work_tiles))

    overwrite = parsed.overwrite

    f = partial(worker,
        process_host=process_host,
        overwrite=overwrite
    )
    pool = Pool()
    logs = []
    try:
        output = pool.map_async(f, work_tiles, callback=logs.extend)
        pool.close()
    except KeyboardInterrupt:
        pool.terminate()
        sys.exit()
    except:
        raise
    finally:
        pool.close()
        pool.join()

    if process_host.config["output_format"] in [
        "GTiff",
        "PNG",
        "PNG_hillshade"
        ]:
        for zoom in process_host.config["zoom_levels"]:
            out_dir = os.path.join(
                process_host.config["output_name"],
                str(zoom)
            )
            out_vrt = os.path.join(
                process_host.config["output_name"],
                (str(zoom)+".vrt")
            )
            command = "gdalbuildvrt -overwrite %s %s" %(
                out_vrt,
                str(out_dir + "/*/*" + process_host.format.extension)
            )
            os.system(command)

    if parsed.log:
        for row in logs:
            if row[1] not in ["ok", "exists"]:
                print row

def worker(tile, process_host, overwrite):
    """
    Worker function running the process depending on the overwrite flag and
    whether the tile exists.
    """
    if not overwrite:
        image_path = process_host.tile_pyramid.format.get_tile_name(
            process_host.config["output_name"],
            tile
        )
        if os.path.isfile(image_path):
            log_message = (tile, "exists", None)
            logger.info(log_message)
            return tile, "exists", None
    try:
        log_message = process_host.save_tile(tile, overwrite)
    except Exception as e:
        log_message = (tile, "failed", traceback.print_exc())

    logger.info(log_message)
    return log_message

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import os
import sys
from flask import request
import re

class Validator:
	def __init__():
		self.requireError = False
		self.maxError 	  = False
		self.minError 	  = False
		self.emailError   = False
		self.integerError = False
		self.stringError  = False

	def validate(dictonary):
		if type(dictonary) not dict:
			raise ValueError('please pass dictionary')

		for dictionary in dictionary:
			print(dictionary)

	#check for required
	def require(self,value,errorMessage = ''):

		if errorMessage:
			self.requireError = errorMessage

		if value == "":
			return self.requireError
		else:
			return ''

	#set maximum value
	def max(self,value,max,errorMessage = ''):
		try:
			val = int(value)
			self.maxError = errorMessage

			if (val > max ):
				return self.maxError
			else:
				return ''

		except ValueError:
			print("it's not an integer")

	#set minimum value
	def min(self,value,min,errorMessage = ''):
		try:
			val = int(value)

			if errorMessage:
				self.minError = errorMessage

			if val < min:
				return self.minError
			else:
				return ''

		except ValueError:
			print("It's not an integer")

	#check for input that email or not
	def email(self,value,errorMessage = ''):
		value = re.group("\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b")

		if errorMessage:
			self.emailError = errorMessage

		if value == "":
			return self.emailError
		else:
			return ''

	#check for integer value
	def integer(self,value,errorMessage =''):
		if errorMessage:
			self.integerError = errorMessage

		if type(value) not int:
			return self.integerError
		else:
			return ''

	#check for string value
	def string(self,value,errorMessage = ''):
		if errorMessage:
			self.stringError = errorMessage

		if type(value) not str:
			return self.stringError
		else:
			return ''<commit_msg>updated validator class<commit_after>import os
import sys
from flask import request
import re

class Validator:
	def __init__():
		self.requireError = False
		self.maxError 	  = False
		self.minError 	  = False
		self.emailError   = False
		self.integerError = False
		self.stringError  = False

	def validate(dictonary):
		if type(dictonary) not dict:
			raise ValueError('please pass dictionary')

		for dictionary in dictionary:
			print(dictionary)

	#check for required
	def require(self,value,errorMessage = ''):

		if errorMessage:
			self.requireError = errorMessage

		if value == "":
			return self.requireError
		else:
			return ''

	#set maximum value
	def max(self,value,max,errorMessage = ''):
		if 	type(max) not int:
			raise ValueError("max value is not an integer")

		val = int(value)
		self.maxError = errorMessage

		if (val > max ):
			return self.maxError
		else:
			return True

	#set minimum value
	def min(self,value,min,errorMessage = ''):

		if type(value) not int:
			raise ValueError('value is not an integer')

		val = int(value)

		if errorMessage:
			self.minError = errorMessage

		if val < min:
			return self.minError
		else:
			return True

	#check for input that email or not
	def email(self,value,errorMessage = ''):
		value = re.group("\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b")

		if errorMessage:
			self.emailError = errorMessage

		if value == "":
			return self.emailError
		else:
			return True

	#check for integer value
	def integer(self,value,errorMessage =''):
		if errorMessage:
			self.integerError = errorMessage

		if type(value) not int:
			return self.integerError
		else:
			return True

	#check for string value
	def string(self,value,errorMessage = ''):
		if errorMessage:
			self.stringError = errorMessage

		if type(value) not str:
			return self.stringError
		else:
			return True

	def between(self,minvalue,maxvalue,errorMessage=''):
		if errorMessag:
			self.errorMessag = errorMessag

		if (type(minvalue) not int) or (type(maxvalue) not int):
			raise ValueError('Min value or max value must be integer')

<|endoftext|>