<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from __future__ import print_function

import cv2
import sys
import time
import traceback

from ikalog.utils import *
from . import scenes

# The IkaLog core engine.
#

class IkaEngine:

    #    def on_game_start(self, context):
    #        self.scn_tower_tracker.reset(context)

    def on_game_individual_result(self, context):
        self.session_close_wdt = context['engine']['msec'] + (20 * 1000)

    def on_result_gears(self, context):
        self.session_close_wdt = context['engine']['msec'] + (1 * 1000)

    def dprint(self, text):
        print(text, file=sys.stderr)

    def call_plugins(self, event_name, debug=False):
        if debug:
            self.dprint('call plug-in hook (%s):' % event_name)

        for op in self.output_plugins:
            if hasattr(op, event_name):
                if debug:
                    self.dprint('Call  %s' % op.__class__.__name__)
                try:
                    getattr(op, event_name)(self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')
            elif hasattr(op, 'onUncatchedEvent'):
                if debug:
                    self.dprint(
                        'call plug-in hook (UncatchedEvent, %s):' % event_name)
                try:
                    getattr(op, 'onUncatchedEvent')(event_name, self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')

    def read_next_frame(self, skip_frames=0):
        for i in range(skip_frames):
            frame, t = self.capture.read()
        frame, t = self.capture.read()

        while frame is None:
            self.call_plugins('on_frame_read_failed')
            if self._stop:
                return None, None
            cv2.waitKey(1000)
            frame, t = self.capture.read()

        self.context['engine']['msec'] = t
        self.context['engine']['frame'] = frame

        self.call_plugins('on_debug_read_next_frame')

        return frame, t

    def stop(self):
        self.call_plugins('on_stop')
        self._stop = True

    def reset(self):
        # Initalize the context
        self.context['game'] = {
            'map': None,
            'rule': None,
            'won': None,
            'players': None,

            'kills': 0,
            'dead': False,
            'death_reasons': {},

            'livesTrack': [],
            'towerTrack': [],
        }
        self.call_plugins('on_game_reset')

    def create_context(self):
        self.context = {
            'engine': {
                'frame': None,
                'service': {
                    'callPlugins': self.call_plugins,
                }
            },
            'scenes': {
            },
            'config': {
            },
            'lobby': {
            }
        }
        self.reset()
        self.session_close_wdt = None

    def session_close(self):
        self.session_close_wdt = None

        self.call_plugins('on_game_session_end')
        self.reset()

    def process_scene(self, scene):
        context = self.context

        scene.new_frame(context)
        r = scene.match(context)
        if r and scene.exclusive_scene:
            print('%s: entering %s' % (self, scene.__class__.__name__))
            while r and (not self._stop):
                frame, t = self.read_next_frame()
                context['engine']['frame'] = frame
                scene.new_frame(context)
                r = scene.match(context)
                if r:
                    scene.analyze(context)
            print('%s: escaping %s' % (self, scene.__class__.__name__))

    def find_scene_object(self, scene_class_name):
        for scene in self.scenes:
            if scene.__class__.__name__ == scene_class_name:
                return scene
        return None

    def process_frame(self):
        context = self.context

        skip_frames = 0
        if (self.capture.from_file and self.capture.fps > 28):
            skip_frames = int(self.capture.fps / 3)

        frame, t = self.read_next_frame(skip_frames=skip_frames)

        if frame is None:
            return False

        context['engine']['inGame'] = self.find_scene_object(
            'GameTimerIcon').match(context)

        self.call_plugins('on_frame_read')

        for scene in self.scenes:
            self.process_scene(scene)

        if self.session_close_wdt is not None:
            if self.session_close_wdt < context['engine']['msec']:
                self.dprint('Watchdog fired. Closing current session')
                self.session_close()

        key = None

        # FixMe: Since on_frame_next and on_key_press has non-standard arguments,
        # self.call_plugins() doesn't work for those.

        for op in self.output_plugins:
            if hasattr(op, "on_frame_next"):
                try:
                    key = op.on_frame_next(context)
                except:
                    pass

        for op in self.output_plugins:
            if hasattr(op, "on_key_press"):
                try:
                    op.on_key_press(context, key)
                except:
                    pass

    def run(self):
        # Main loop.
        while not self._stop:
            if self._pause:
                time.sleep(0.5)
                continue

            try:
                self.process_frame()
            except EOFError:
                # EOF. Close session if close_session_at_eof is set.
                if self.close_session_at_eof:
                    if self.session_close_wdt is not None:
                        self.dprint('Closing current session at EOF')
                        self.session_close()
                raise

        cv2.destroyAllWindows()

    def set_capture(self, capture):
        self.capture = capture

    def set_plugins(self, plugins):
        self.output_plugins = [self]
        self.output_plugins.extend(plugins)

    def pause(self, pause):
        self._pause = pause

    def _initialize_scenes(self):
        self.scenes = [
            scenes.GameTimerIcon(self),
            scenes.GameStart(self),
            scenes.GameGoSign(self),
            scenes.GameKill(self),
            scenes.GameDead(self),
            scenes.GameOutOfBound(self),
            scenes.GameFinish(self),
            scenes.ResultJudge(self),
            scenes.PaintScoreTracker(self),
            scenes.ObjectiveTracker(self),
            scenes.ResultDetail(self),
            scenes.ResultUdemae(self),
            scenes.ResultGears(self),
            scenes.ResultFesta(self),
            scenes.Lobby(self),
        ]

    def __init__(self):
        self._initialize_scenes()

        self.output_plugins = [self]
        self.last_capture = time.time() - 100

        self._stop = False
        self._pause = True

        self.close_session_at_eof = False

        self.create_context()
<commit_msg>ikalog/engine: set skip_frames to 10% (100ms) of framerate.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from __future__ import print_function

import cv2
import sys
import time
import traceback

from ikalog.utils import *
from . import scenes

# The IkaLog core engine.
#

class IkaEngine:

    #    def on_game_start(self, context):
    #        self.scn_tower_tracker.reset(context)

    def on_game_individual_result(self, context):
        self.session_close_wdt = context['engine']['msec'] + (20 * 1000)

    def on_result_gears(self, context):
        self.session_close_wdt = context['engine']['msec'] + (1 * 1000)

    def dprint(self, text):
        print(text, file=sys.stderr)

    def call_plugins(self, event_name, debug=False):
        if debug:
            self.dprint('call plug-in hook (%s):' % event_name)

        for op in self.output_plugins:
            if hasattr(op, event_name):
                if debug:
                    self.dprint('Call  %s' % op.__class__.__name__)
                try:
                    getattr(op, event_name)(self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')
            elif hasattr(op, 'onUncatchedEvent'):
                if debug:
                    self.dprint(
                        'call plug-in hook (UncatchedEvent, %s):' % event_name)
                try:
                    getattr(op, 'onUncatchedEvent')(event_name, self.context)
                except:
                    self.dprint('%s.%s() raised a exception >>>>' %
                                (op.__class__.__name__, event_name))
                    self.dprint(traceback.format_exc())
                    self.dprint('<<<<<')

    def read_next_frame(self, skip_frames=0):
        for i in range(skip_frames):
            frame, t = self.capture.read()
        frame, t = self.capture.read()

        while frame is None:
            self.call_plugins('on_frame_read_failed')
            if self._stop:
                return None, None
            cv2.waitKey(1000)
            frame, t = self.capture.read()

        self.context['engine']['msec'] = t
        self.context['engine']['frame'] = frame

        self.call_plugins('on_debug_read_next_frame')

        return frame, t

    def stop(self):
        self.call_plugins('on_stop')
        self._stop = True

    def reset(self):
        # Initalize the context
        self.context['game'] = {
            'map': None,
            'rule': None,
            'won': None,
            'players': None,

            'kills': 0,
            'dead': False,
            'death_reasons': {},

            'livesTrack': [],
            'towerTrack': [],
        }
        self.call_plugins('on_game_reset')

    def create_context(self):
        self.context = {
            'engine': {
                'frame': None,
                'service': {
                    'callPlugins': self.call_plugins,
                }
            },
            'scenes': {
            },
            'config': {
            },
            'lobby': {
            }
        }
        self.reset()
        self.session_close_wdt = None

    def session_close(self):
        self.session_close_wdt = None

        self.call_plugins('on_game_session_end')
        self.reset()

    def process_scene(self, scene):
        context = self.context

        scene.new_frame(context)
        r = scene.match(context)
        if r and scene.exclusive_scene:
            print('%s: entering %s' % (self, scene.__class__.__name__))
            while r and (not self._stop):
                frame, t = self.read_next_frame()
                context['engine']['frame'] = frame
                scene.new_frame(context)
                r = scene.match(context)
                if r:
                    scene.analyze(context)
            print('%s: escaping %s' % (self, scene.__class__.__name__))

    def find_scene_object(self, scene_class_name):
        for scene in self.scenes:
            if scene.__class__.__name__ == scene_class_name:
                return scene
        return None

    def process_frame(self):
        context = self.context

        skip_frames = 0
        if (self.capture.from_file and self.capture.fps > 28):
            skip_frames = int(self.capture.fps * 0.1)

        frame, t = self.read_next_frame(skip_frames=skip_frames)

        if frame is None:
            return False

        context['engine']['inGame'] = self.find_scene_object(
            'GameTimerIcon').match(context)

        self.call_plugins('on_frame_read')

        for scene in self.scenes:
            self.process_scene(scene)

        if self.session_close_wdt is not None:
            if self.session_close_wdt < context['engine']['msec']:
                self.dprint('Watchdog fired. Closing current session')
                self.session_close()

        key = None

        # FixMe: Since on_frame_next and on_key_press has non-standard arguments,
        # self.call_plugins() doesn't work for those.

        for op in self.output_plugins:
            if hasattr(op, "on_frame_next"):
                try:
                    key = op.on_frame_next(context)
                except:
                    pass

        for op in self.output_plugins:
            if hasattr(op, "on_key_press"):
                try:
                    op.on_key_press(context, key)
                except:
                    pass

    def run(self):
        # Main loop.
        while not self._stop:
            if self._pause:
                time.sleep(0.5)
                continue

            try:
                self.process_frame()
            except EOFError:
                # EOF. Close session if close_session_at_eof is set.
                if self.close_session_at_eof:
                    if self.session_close_wdt is not None:
                        self.dprint('Closing current session at EOF')
                        self.session_close()
                raise

        cv2.destroyAllWindows()

    def set_capture(self, capture):
        self.capture = capture

    def set_plugins(self, plugins):
        self.output_plugins = [self]
        self.output_plugins.extend(plugins)

    def pause(self, pause):
        self._pause = pause

    def _initialize_scenes(self):
        self.scenes = [
            scenes.GameTimerIcon(self),
            scenes.GameStart(self),
            scenes.GameGoSign(self),
            scenes.GameKill(self),
            scenes.GameDead(self),
            scenes.GameOutOfBound(self),
            scenes.GameFinish(self),
            scenes.ResultJudge(self),
            scenes.PaintScoreTracker(self),
            scenes.ObjectiveTracker(self),
            scenes.ResultDetail(self),
            scenes.ResultUdemae(self),
            scenes.ResultGears(self),
            scenes.ResultFesta(self),
            scenes.Lobby(self),
        ]

    def __init__(self):
        self._initialize_scenes()

        self.output_plugins = [self]
        self.last_capture = time.time() - 100

        self._stop = False
        self._pause = True

        self.close_session_at_eof = False

        self.create_context()
<|endoftext|>


<commit_before>"""Classes to configure and launch jobs from a wmt-exe environment."""

import os
import sys
import stat
import subprocess
from types import StringTypes

class Launcher(object):
    """Job launcher for a wmt-exe environment.

    Parameters
    ----------
    sim_id : str
        A unique UUID for the job.
    server_url : str or None, optional
        The URL of the WMT API server from which the job was submitted.

    Attributes
    ----------
    launch_dir : str
        The working directory from which the job is started.
    script_path : str
        Path to launch script.
    sim_id : str
        A unique UUID for the job.
    server_url : str or None
        The URL of the WMT API server from which the job was submitted.

    """
    launch_dir = '~/.wmt'
    _script = "{slave_command}"
    _extra_args = []

    def __init__(self, sim_id, server_url=None):
        self.sim_id = sim_id
        self.server_url = server_url
        self.script_path = os.path.expanduser(
            os.path.join(self.launch_dir,
                         '%s.sh' % self.sim_id))

    def before_launch(self, **kwds):
        """Perform actions before launching job.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        with open(self.script_path, 'w') as f:
            f.write(self.script(**kwds))
        os.chmod(self.script_path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR)

    def after_launch(self, **kwds):
        """Perform actions after launching job.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        pass

    def after_success(self, **kwds):
        """Perform actions after job completes.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        pass

    def run(self, **kwds):
        """Perform job setup, launch, and teardown actions.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        self.before_launch(**kwds)

        try:
            self.launch(**kwds)
        except subprocess.CalledProcessError:
            raise
        else:
            self.after_success(**kwds)
        finally:
            self.after_launch(**kwds)

    def launch(self, **kwds):
        """Launch job with launch command.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        subprocess.check_output(self.launch_command(**kwds), env={})

    def launch_command(self, **kwds):
        """Path to launch script.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch command to execute.

        """
        return self.script_path

    def slave_command(self, extra_args=None):
        """Create the `wmt-slave` command.

        Parameters
        ----------
        extra_args : str, optional
            Additional arguments.

        Returns
        -------
        str
            The slave command to execute.

        """
        import shlex
        from pipes import quote

        wmt_slave = os.path.join(sys.prefix, 'bin', 'wmt-slave')
        command = [wmt_slave, quote(self.sim_id)] + self._extra_args

        if self.server_url:
            command += ['--server-url={}'.format(self.server_url)]

        if extra_args:
            if isinstance(extra_args, StringTypes):
                extra_args = shlex.split(extra_args)
            command += [quote(arg) for arg in extra_args]

        return ' '.join(command)

    def script(self, **kwds):
        """Generate the launch script.

        Parameters
        ----------
        *kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch script to be written to a file.

        """
        return self._script.format(slave_command=self.slave_command(**kwds))

class QsubLauncher(Launcher):
    """WMT job launcher for a PBS scheduler."""
    _script = """
#! /bin/bash
#PBS -q debug
#PBS -l mem=10gb
#PBS -j oe
#PBS -k oe

cd $TMPDIR

{slave_command}
""".lstrip()
    _extra_args = ['--exec-dir=$TMPDIR']

    def launch_command(self, **kwds):
        """Path to launch script.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch command to execute.

        """
        return ['/opt/torque/bin/qsub', '-o', self.launch_dir,
                self.script_path]

class SbatchLauncher(Launcher):
    """WMT job launcher for a SLURM scheduler."""
    _script = """
#!/usr/bin/env bash
#SBATCH --qos=blanca-csdms
#SBATCH --job-name=wmt
#SBATCH --output=wmt-%j.out

{slave_command}
""".lstrip()

    def launch_command(self, **kwds):
        """Path to launch script.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch command to execute.

        """
        return 'module load slurm/blanca && sbatch {}'.format(self.script_path)

    def launch(self, **kwds):
        """Launch job with launch command.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        subprocess.check_output(self.launch_command(**kwds), shell=True)

class BashLauncher(Launcher):
    """WMT job launcher for a bash environment."""
    _script = """
#! /bin/bash

export PATH={wmt_path}

{slave_command}
""".lstrip()

    def prepend_path(self):
        """Places the `bin` directory of executor at the front of the path."""
        return os.pathsep.join([os.path.join(sys.prefix, 'bin'),
                                os.environ['PATH']])

    def script(self, **kwds):
        """Generate the launch script.

        Parameters
        ----------
        *kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch script to be written to a file.

        """
        return self._script.format(wmt_path=self.prepend_path(),
                                   slave_command=self.slave_command(**kwds))
<commit_msg>Add run script<commit_after>"""Classes to configure and launch jobs from a wmt-exe environment."""

import os
import sys
import stat
import subprocess
from types import StringTypes

class Launcher(object):
    """Job launcher for a wmt-exe environment.

    Parameters
    ----------
    sim_id : str
        A unique UUID for the job.
    server_url : str or None, optional
        The URL of the WMT API server from which the job was submitted.

    Attributes
    ----------
    launch_dir : str
        The working directory from which the job is started.
    script_path : str
        Path to launch script.
    sim_id : str
        A unique UUID for the job.
    server_url : str or None
        The URL of the WMT API server from which the job was submitted.

    """
    launch_dir = '~/.wmt'
    _script = "{slave_command}"
    _extra_args = []

    def __init__(self, sim_id, server_url=None):
        self.sim_id = sim_id
        self.server_url = server_url
        self.script_path = os.path.expanduser(
            os.path.join(self.launch_dir,
                         '%s.sh' % self.sim_id))

    def before_launch(self, **kwds):
        """Perform actions before launching job.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        with open(self.script_path, 'w') as f:
            f.write(self.script(**kwds))
        os.chmod(self.script_path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR)

    def after_launch(self, **kwds):
        """Perform actions after launching job.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        pass

    def after_success(self, **kwds):
        """Perform actions after job completes.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        pass

    def run(self, **kwds):
        """Perform job setup, launch, and teardown actions.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        self.before_launch(**kwds)

        try:
            self.launch(**kwds)
        except subprocess.CalledProcessError:
            raise
        else:
            self.after_success(**kwds)
        finally:
            self.after_launch(**kwds)

    def launch(self, **kwds):
        """Launch job with launch command.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        """
        subprocess.check_output(self.launch_command(**kwds), env={})

    def launch_command(self, **kwds):
        """Path to launch script.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch command to execute.

        """
        return self.script_path

    def slave_command(self, extra_args=None):
        """Create the `wmt-slave` command.

        Parameters
        ----------
        extra_args : str, optional
            Additional arguments.

        Returns
        -------
        str
            The slave command to execute.

        """
        import shlex
        from pipes import quote

        wmt_slave = os.path.join(sys.prefix, 'bin', 'wmt-slave')
        command = [wmt_slave, quote(self.sim_id)] + self._extra_args

        if self.server_url:
            command += ['--server-url={}'.format(self.server_url)]

        if extra_args:
            if isinstance(extra_args, StringTypes):
                extra_args = shlex.split(extra_args)
            command += [quote(arg) for arg in extra_args]

        return ' '.join(command)

    def script(self, **kwds):
        """Generate the launch script.

        Parameters
        ----------
        *kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch script to be written to a file.

        """
        return self._script.format(slave_command=self.slave_command(**kwds))

class QsubLauncher(Launcher):
    """WMT job launcher for a PBS scheduler."""
    _script = """
#! /bin/bash
#PBS -q debug
#PBS -l mem=10gb
#PBS -j oe
#PBS -k oe

cd $TMPDIR

{slave_command}
""".lstrip()
    _extra_args = ['--exec-dir=$TMPDIR']

    def launch_command(self, **kwds):
        """Path to launch script.

        Parameters
        ----------
        **kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch command to execute.

        """
        return ['/opt/torque/bin/qsub', '-o', self.launch_dir,
                self.script_path]

class SbatchLauncher(Launcher):
    """WMT job launcher for a SLURM scheduler."""
    _script = """
#!/usr/bin/env bash
#SBATCH --qos=blanca-csdms
#SBATCH --job-name=wmt
#SBATCH --output=wmt-%j.out

{slave_command}
""".lstrip()
    _run_script = """
#!/usr/bin/env bash

module load slurm/blanca
sbatch {script_path}
""".lstrip()

    def __init__(self, *args, **kwds):
        Launcher.__init__(self, *args, **kwds)
        self.run_script_path = os.path.expanduser(
            os.path.join(self.launch_dir,
                         '%s.run.sh' % self.sim_id))

    def before_launch(self, **kwds):
        Launcher.before_launch(self, **kwds)
        with open(self.run_script_path, 'w') as f:
            f.write(self.run_script(**kwds))
        os.chmod(self.run_script_path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR)

    def run_script(self, **kwds):
        return self._run_script.format(script_path=self.script_path)

    def launch_command(self, **kwds):
        return self.run_script_path

    def launch(self, **kwds):
        os.system(self.launch_command())

class BashLauncher(Launcher):
    """WMT job launcher for a bash environment."""
    _script = """
#! /bin/bash

export PATH={wmt_path}

{slave_command}
""".lstrip()

    def prepend_path(self):
        """Places the `bin` directory of executor at the front of the path."""
        return os.pathsep.join([os.path.join(sys.prefix, 'bin'),
                                os.environ['PATH']])

    def script(self, **kwds):
        """Generate the launch script.

        Parameters
        ----------
        *kwds
            Arbitrary keyword arguments.

        Returns
        -------
        str
            The launch script to be written to a file.

        """
        return self._script.format(wmt_path=self.prepend_path(),
                                   slave_command=self.slave_command(**kwds))
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Time related utilities and helper functions.
"""

import calendar
import datetime

import iso8601

# ISO 8601 extended time format with microseconds
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND

def isotime(at=None, subsecond=False):
    """Stringify time in ISO 8601 format."""
    if not at:
        at = utcnow()
    st = at.strftime(_ISO8601_TIME_FORMAT
                     if not subsecond
                     else _ISO8601_TIME_FORMAT_SUBSECOND)
    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
    st += ('Z' if tz == 'UTC' else tz)
    return st

def parse_isotime(timestr):
    """Parse time from ISO 8601 format."""
    try:
        return iso8601.parse_date(timestr)
    except iso8601.ParseError as e:
        raise ValueError(e.message)
    except TypeError as e:
        raise ValueError(e.message)

def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
    """Returns formatted utcnow."""
    if not at:
        at = utcnow()
    return at.strftime(fmt)

def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
    """Turn a formatted time back into a datetime."""
    return datetime.datetime.strptime(timestr, fmt)

def normalize_time(timestamp):
    """Normalize time in arbitrary timezone to UTC naive object."""
    offset = timestamp.utcoffset()
    if offset is None:
        return timestamp
    return timestamp.replace(tzinfo=None) - offset

def is_older_than(before, seconds):
    """Return True if before is older than seconds."""
    if isinstance(before, basestring):
        before = parse_strtime(before).replace(tzinfo=None)
    return utcnow() - before > datetime.timedelta(seconds=seconds)

def is_newer_than(after, seconds):
    """Return True if after is newer than seconds."""
    if isinstance(after, basestring):
        after = parse_strtime(after).replace(tzinfo=None)
    return after - utcnow() > datetime.timedelta(seconds=seconds)

def utcnow_ts():
    """Timestamp version of our utcnow function."""
    return calendar.timegm(utcnow().timetuple())

def utcnow():
    """Overridable version of utils.utcnow."""
    if utcnow.override_time:
        try:
            return utcnow.override_time.pop(0)
        except AttributeError:
            return utcnow.override_time
    return datetime.datetime.utcnow()

def iso8601_from_timestamp(timestamp):
    """Returns a iso8601 formated date from timestamp."""
    return isotime(datetime.datetime.utcfromtimestamp(timestamp))

utcnow.override_time = None

def set_time_override(override_time=datetime.datetime.utcnow()):
    """Overrides utils.utcnow.

    Make it return a constant time or a list thereof, one at a time.
    """
    utcnow.override_time = override_time

def advance_time_delta(timedelta):
    """Advance overridden time using a datetime.timedelta."""
    assert(not utcnow.override_time is None)
    try:
        for dt in utcnow.override_time:
            dt += timedelta
    except TypeError:
        utcnow.override_time += timedelta

def advance_time_seconds(seconds):
    """Advance overridden time by seconds."""
    advance_time_delta(datetime.timedelta(0, seconds))

def clear_time_override():
    """Remove the overridden time."""
    utcnow.override_time = None

def marshall_now(now=None):
    """Make an rpc-safe datetime with microseconds.

    Note: tzinfo is stripped, but not required for relative times.
    """
    if not now:
        now = utcnow()
    return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
                minute=now.minute, second=now.second,
                microsecond=now.microsecond)

def unmarshall_time(tyme):
    """Unmarshall a datetime dict."""
    return datetime.datetime(day=tyme['day'],
                             month=tyme['month'],
                             year=tyme['year'],
                             hour=tyme['hour'],
                             minute=tyme['minute'],
                             second=tyme['second'],
                             microsecond=tyme['microsecond'])

def delta_seconds(before, after):
    """Return the difference between two timing objects.

    Compute the difference in seconds between two date, time, or
    datetime objects (as a float, to microsecond resolution).
    """
    delta = after - before
    try:
        return delta.total_seconds()
    except AttributeError:
        return ((delta.days * 24 * 3600) + delta.seconds +
                float(delta.microseconds) / (10 ** 6))

def is_soon(dt, window):
    """Determines if time is going to happen in the next window seconds.

    :params dt: the time
    :params window: minimum seconds to remain to consider the time not soon

    :return: True if expiration is within the given duration
    """
    soon = (utcnow() + datetime.timedelta(seconds=window))
    return normalize_time(dt) <= soon
<commit_msg>Sync module timeutils from Oslo<commit_after># Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Time related utilities and helper functions.
"""

import calendar
import datetime
import time

import iso8601
import six

# ISO 8601 extended time format with microseconds
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND

def isotime(at=None, subsecond=False):
    """Stringify time in ISO 8601 format."""
    if not at:
        at = utcnow()
    st = at.strftime(_ISO8601_TIME_FORMAT
                     if not subsecond
                     else _ISO8601_TIME_FORMAT_SUBSECOND)
    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
    st += ('Z' if tz == 'UTC' else tz)
    return st

def parse_isotime(timestr):
    """Parse time from ISO 8601 format."""
    try:
        return iso8601.parse_date(timestr)
    except iso8601.ParseError as e:
        raise ValueError(six.text_type(e))
    except TypeError as e:
        raise ValueError(six.text_type(e))

def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
    """Returns formatted utcnow."""
    if not at:
        at = utcnow()
    return at.strftime(fmt)

def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
    """Turn a formatted time back into a datetime."""
    return datetime.datetime.strptime(timestr, fmt)

def normalize_time(timestamp):
    """Normalize time in arbitrary timezone to UTC naive object."""
    offset = timestamp.utcoffset()
    if offset is None:
        return timestamp
    return timestamp.replace(tzinfo=None) - offset

def is_older_than(before, seconds):
    """Return True if before is older than seconds."""
    if isinstance(before, six.string_types):
        before = parse_strtime(before).replace(tzinfo=None)
    else:
        before = before.replace(tzinfo=None)

    return utcnow() - before > datetime.timedelta(seconds=seconds)

def is_newer_than(after, seconds):
    """Return True if after is newer than seconds."""
    if isinstance(after, six.string_types):
        after = parse_strtime(after).replace(tzinfo=None)
    else:
        after = after.replace(tzinfo=None)

    return after - utcnow() > datetime.timedelta(seconds=seconds)

def utcnow_ts():
    """Timestamp version of our utcnow function."""
    if utcnow.override_time is None:
        # NOTE(kgriffs): This is several times faster
        # than going through calendar.timegm(...)
        return int(time.time())

    return calendar.timegm(utcnow().timetuple())

def utcnow():
    """Overridable version of utils.utcnow."""
    if utcnow.override_time:
        try:
            return utcnow.override_time.pop(0)
        except AttributeError:
            return utcnow.override_time
    return datetime.datetime.utcnow()

def iso8601_from_timestamp(timestamp):
    """Returns a iso8601 formated date from timestamp."""
    return isotime(datetime.datetime.utcfromtimestamp(timestamp))

utcnow.override_time = None

def set_time_override(override_time=None):
    """Overrides utils.utcnow.

    Make it return a constant time or a list thereof, one at a time.

    :param override_time: datetime instance or list thereof. If not
                          given, defaults to the current UTC time.
    """
    utcnow.override_time = override_time or datetime.datetime.utcnow()

def advance_time_delta(timedelta):
    """Advance overridden time using a datetime.timedelta."""
    assert(not utcnow.override_time is None)
    try:
        for dt in utcnow.override_time:
            dt += timedelta
    except TypeError:
        utcnow.override_time += timedelta

def advance_time_seconds(seconds):
    """Advance overridden time by seconds."""
    advance_time_delta(datetime.timedelta(0, seconds))

def clear_time_override():
    """Remove the overridden time."""
    utcnow.override_time = None

def marshall_now(now=None):
    """Make an rpc-safe datetime with microseconds.

    Note: tzinfo is stripped, but not required for relative times.
    """
    if not now:
        now = utcnow()
    return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
                minute=now.minute, second=now.second,
                microsecond=now.microsecond)

def unmarshall_time(tyme):
    """Unmarshall a datetime dict."""
    return datetime.datetime(day=tyme['day'],
                             month=tyme['month'],
                             year=tyme['year'],
                             hour=tyme['hour'],
                             minute=tyme['minute'],
                             second=tyme['second'],
                             microsecond=tyme['microsecond'])

def delta_seconds(before, after):
    """Return the difference between two timing objects.

    Compute the difference in seconds between two date, time, or
    datetime objects (as a float, to microsecond resolution).
    """
    delta = after - before
    return total_seconds(delta)

def total_seconds(delta):
    """Return the total seconds of datetime.timedelta object.

    Compute total seconds of datetime.timedelta, datetime.timedelta
    doesn't have method total_seconds in Python2.6, calculate it manually.
    """
    try:
        return delta.total_seconds()
    except AttributeError:
        return ((delta.days * 24 * 3600) + delta.seconds +
                float(delta.microseconds) / (10 ** 6))

def is_soon(dt, window):
    """Determines if time is going to happen in the next window seconds.

    :param dt: the time
    :param window: minimum seconds to remain to consider the time not soon

    :return: True if expiration is within the given duration
    """
    soon = (utcnow() + datetime.timedelta(seconds=window))
    return normalize_time(dt) <= soon
<|endoftext|>


<commit_before>import datetime

from django.db import models
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.utils.translation import ugettext as _
from django.conf import settings

from nani.models import TranslatableModel, TranslatedFields

from apps.survey.models import SurveyUser

# A short word on terminology:
# "Reminder" may refer to a NewsLetter object, or simply a placeholder
# that's based on the is_default_reminder NewsLetterTemplate

NO_INTERVAL = -1
WEEKLY_WITH_BATCHES = -2

class UserReminderInfo(models.Model):
    user = models.ForeignKey(User, unique=True)
    last_reminder = models.DateTimeField(null=True, blank=True)
    active = models.BooleanField()
    language = models.CharField(max_length=5, blank=True, null=True)

    def __unicode__(self):
        return self.user.username

    def get_language(self):
        if not self.language:
            return settings.LANGUAGE_CODE
        return self.language

class ReminderSettings(models.Model):
    site = models.OneToOneField(Site)
    send_reminders = models.BooleanField(_("Send reminders"), help_text=_("Check this box to send reminders"))
    interval = models.IntegerField(_("Interval"), choices=((7 ,_("Weekly")), (14,_("Bi-weekly")), (NO_INTERVAL, _("Don't send reminders at a fixed interval")), (WEEKLY_WITH_BATCHES, "Send exactly 7 days after the last action was taken.")), null=True, blank=True)
    begin_date = models.DateTimeField(_("Begin date"), help_text="Date & time of the first reminder and point of reference for subsequent reminders; (Time zone: %s)" % settings.TIME_ZONE, null=True, blank=True)
    batch_size = models.IntegerField("Batch size", null=True, blank=True, help_text="Batch size determines the max. sent emails per call to 'reminder_send'; choose in coordinance with you r crontab interval and total users; Leave empty to not have any maximum")
    currently_sending = models.BooleanField("Currently sending", help_text="This indicates if the reminders are being sent right now. Don't tick this box unless you absolutely know what you're doing", default=False)
    last_process_started_date = models.DateTimeField("Last process started at", help_text="This indicates if the reminders are being sent right now. Don't change this value unless you absolutely know what you're doing")

    def __unicode__(self):
        return _(u"Reminder settings")

    def get_interval(self):
        if self.interval == WEEKLY_WITH_BATCHES:
            return 7
        return self.interval

class NewsLetterTemplate(TranslatableModel):
    is_default_reminder = models.BooleanField(_("Is default reminder"), help_text=_("If this option is checked this template is the standard template for reminder emails."))
    is_default_newsitem = models.BooleanField(_("Is default newsitem"), help_text=_("If this option is checked this template is the standard template for new news items."))
    sender_email = models.EmailField(_("Sender email"), help_text="Only use email addresses for your main domain to ensure deliverability")
    sender_name = models.CharField(_("Sender name"), max_length=255)

    translations = TranslatedFields(
        subject = models.CharField(max_length=255),
        message = models.TextField(help_text="The strings {{ url }} and {{ unsubscribe_url }} may be used to refer to the profile url and unsubscribe url."),
    )

    def __unicode__(self):
        return self.subject

class NewsLetter(TranslatableModel):
    date = models.DateTimeField(_("Date"), unique=True, choices=[])

    sender_email = models.EmailField(_("Sender email"), help_text="Only use email addresses for your main domain to ensure deliverability")
    sender_name = models.CharField(_("Sender name"), max_length=255)

    published = models.BooleanField("Is published", help_text="Uncheck this box to postpone sending of this newsletter until the box is checked.", default=True)

    translations = TranslatedFields(
        subject = models.CharField(max_length=255),
        message = models.TextField(help_text="The strings {{ url }} and {{ unsubscribe_url }} may be used to refer to the profile url and unsubscribe url."),
    )

    def __unicode__(self):
        return self.subject

    class Meta:
        ordering = ("-date",)

class MockNewsLetter(object):
    def __init__(self):
        self.date = self.sender_email= self.sender_name = self.subject = self.message = None

class ReminderError(models.Model):
    timestamp = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(User)
    message = models.CharField(max_length=255)
    traceback = models.TextField()

    def __unicode__(self):
        return self.message

    class Meta:
        ordering = ("-timestamp",)

    def email(self):
        return self.user.email

def get_settings():
    if ReminderSettings.objects.count() == 0:
        return None
    return ReminderSettings.objects.all()[0]

def get_upcoming_dates(now):
    settings = get_settings()
    if not settings or not settings.send_reminders or not settings.begin_date:
        raise StopIteration()

    if settings.get_interval() < 0:
        yield now, "Current newsletter"
        raise StopIteration()

    to_yield = 7
    current = settings.begin_date

    while to_yield > 0:
        if current >= now - datetime.timedelta(2 * settings.get_interval()):
            diff = current - now
            days = abs(diff.days) % 7
            weeks = abs(diff.days) / 7
            if diff.days > 0:
                if weeks == 0:
                    yield current, _("%(current)s (in %(days)s days)") % locals()
                else:
                    yield current, _("%(current)s (in %(weeks)s weeks)") % locals()
            else:
                if weeks == 0:
                    yield current, _("%(current)s (%(days)s days ago)") % locals()
                else:
                    yield current, _("%(current)s (%(weeks)s weeks ago)") % locals()
            to_yield -= 1
        current += datetime.timedelta(settings.get_interval())

def get_default_for_reminder(language):
    if NewsLetterTemplate.objects.language(language).filter(is_default_reminder=True).count() == 0:
        return None
    return NewsLetterTemplate.objects.language(language).filter(is_default_reminder=True)[0]

def get_default_for_newsitem(language):
    if NewsLetterTemplate.objects.language(language).filter(is_default_newsitem=True).count() == 0:
        return None
    return NewsLetterTemplate.objects.language(language).filter(is_default_newsitem=True)[0]

def get_prev_reminder_date(now, published=True):
    """Returns the date of the previous reminder or None if there's no
    such date"""

    settings = get_settings()

    if not settings or not settings.send_reminders or not settings.begin_date or now < settings.begin_date:
        return None

    if settings.interval == NO_INTERVAL:
        qs = NewsLetter.objects.filter(date__lte=now).exclude(date__gt=now).order_by("-date")
        if published:
            qs = qs.filter(published=published)
        qs = list(qs)
        if len(qs) == 0:
            return None
        return qs[0].date

    current = settings.begin_date
    prev = settings.begin_date

    while True:
        if current >= now:
            return prev
        prev = current
        current += datetime.timedelta(settings.get_interval())

def get_prev_reminder(now, published=True):
    """Returns the reminder (newsletter/tempate) to send at a given moment
    as a dict with languages as keys, or None if there is no such reminder"""
    def p(qs):
        if published:
            return qs.filter(published=published)
        return qs

    prev_date = get_prev_reminder_date(now, published)
    if prev_date is None:
        return None

    if p(NewsLetter.objects.filter(date=prev_date)).count():
        result = {}
        for language, name in settings.LANGUAGES:
            if p(NewsLetter.objects.language(language).filter(date=prev_date)):
                result[language] = p(NewsLetter.objects.language(language)).get(date=prev_date)
        return result

    result = {}
    for language, name in settings.LANGUAGES:
        template = get_default_for_reminder(language)
        if template is None:
            continue

        newsletter = MockNewsLetter()
        newsletter.date = prev_date
        newsletter.sender_email = template.sender_email
        newsletter.sender_name = template.sender_name
        newsletter.subject = template.subject
        newsletter.message = template.message
        result[language] = newsletter

    return result

def get_reminders_for_users(now, users):
    reminder_dict = get_prev_reminder(now)
    if not reminder_dict:
        raise StopIteration()

    batch_size = get_settings().batch_size if get_settings() else None

    yielded = 0
    for user in users:
        if batch_size and yielded >= batch_size:
            raise StopIteration

        info, _ = UserReminderInfo.objects.get_or_create(user=user, defaults={'active': True, 'last_reminder': user.date_joined})

        if not info.active:
            continue

        language = info.get_language()
        if not language in reminder_dict:
            language = settings.LANGUAGE_CODE

        reminder = reminder_dict[language]

        if info.last_reminder is None:
            yield user, reminder, language
            yielded += 1
            continue

        if get_settings() and get_settings().interval == WEEKLY_WITH_BATCHES:
            survey_users = SurveyUser.objects.filter(user=user, deleted=False)
            if not survey_users.count():
                survey_user = SurveyUser.objects.create(user=user, name=user.username)
                survey_users = SurveyUser.objects.filter(user=user)

            last_action = (now - max(su.get_last_weekly_survey_date() for su in survey_users)).days
            if info.last_reminder and info.last_reminder > last_action:
                last_action = info.last_reminder

            last_action_long_ago_enough = last_action >= 7

            if not last_action_long_ago_enough:
                continue

        if info.last_reminder < reminder.date:
            yield user, reminder, language
            yielded += 1

<commit_msg>Bugfix<commit_after>import datetime

from django.db import models
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.utils.translation import ugettext as _
from django.conf import settings

from nani.models import TranslatableModel, TranslatedFields

from apps.survey.models import SurveyUser

# A short word on terminology:
# "Reminder" may refer to a NewsLetter object, or simply a placeholder
# that's based on the is_default_reminder NewsLetterTemplate

NO_INTERVAL = -1
WEEKLY_WITH_BATCHES = -2

class UserReminderInfo(models.Model):
    user = models.ForeignKey(User, unique=True)
    last_reminder = models.DateTimeField(null=True, blank=True)
    active = models.BooleanField()
    language = models.CharField(max_length=5, blank=True, null=True)

    def __unicode__(self):
        return self.user.username

    def get_language(self):
        if not self.language:
            return settings.LANGUAGE_CODE
        return self.language

class ReminderSettings(models.Model):
    site = models.OneToOneField(Site)
    send_reminders = models.BooleanField(_("Send reminders"), help_text=_("Check this box to send reminders"))
    interval = models.IntegerField(_("Interval"), choices=((7 ,_("Weekly")), (14,_("Bi-weekly")), (NO_INTERVAL, _("Don't send reminders at a fixed interval")), (WEEKLY_WITH_BATCHES, "Send exactly 7 days after the last action was taken.")), null=True, blank=True)
    begin_date = models.DateTimeField(_("Begin date"), help_text="Date & time of the first reminder and point of reference for subsequent reminders; (Time zone: %s)" % settings.TIME_ZONE, null=True, blank=True)
    batch_size = models.IntegerField("Batch size", null=True, blank=True, help_text="Batch size determines the max. sent emails per call to 'reminder_send'; choose in coordinance with you r crontab interval and total users; Leave empty to not have any maximum")
    currently_sending = models.BooleanField("Currently sending", help_text="This indicates if the reminders are being sent right now. Don't tick this box unless you absolutely know what you're doing", default=False)
    last_process_started_date = models.DateTimeField("Last process started at", help_text="This indicates if the reminders are being sent right now. Don't change this value unless you absolutely know what you're doing")

    def __unicode__(self):
        return _(u"Reminder settings")

    def get_interval(self):
        if self.interval == WEEKLY_WITH_BATCHES:
            return 7
        return self.interval

class NewsLetterTemplate(TranslatableModel):
    is_default_reminder = models.BooleanField(_("Is default reminder"), help_text=_("If this option is checked this template is the standard template for reminder emails."))
    is_default_newsitem = models.BooleanField(_("Is default newsitem"), help_text=_("If this option is checked this template is the standard template for new news items."))
    sender_email = models.EmailField(_("Sender email"), help_text="Only use email addresses for your main domain to ensure deliverability")
    sender_name = models.CharField(_("Sender name"), max_length=255)

    translations = TranslatedFields(
        subject = models.CharField(max_length=255),
        message = models.TextField(help_text="The strings {{ url }} and {{ unsubscribe_url }} may be used to refer to the profile url and unsubscribe url."),
    )

    def __unicode__(self):
        return self.subject

class NewsLetter(TranslatableModel):
    date = models.DateTimeField(_("Date"), unique=True, choices=[])

    sender_email = models.EmailField(_("Sender email"), help_text="Only use email addresses for your main domain to ensure deliverability")
    sender_name = models.CharField(_("Sender name"), max_length=255)

    published = models.BooleanField("Is published", help_text="Uncheck this box to postpone sending of this newsletter until the box is checked.", default=True)

    translations = TranslatedFields(
        subject = models.CharField(max_length=255),
        message = models.TextField(help_text="The strings {{ url }} and {{ unsubscribe_url }} may be used to refer to the profile url and unsubscribe url."),
    )

    def __unicode__(self):
        return self.subject

    class Meta:
        ordering = ("-date",)

class MockNewsLetter(object):
    def __init__(self):
        self.date = self.sender_email= self.sender_name = self.subject = self.message = None

class ReminderError(models.Model):
    timestamp = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(User)
    message = models.CharField(max_length=255)
    traceback = models.TextField()

    def __unicode__(self):
        return self.message

    class Meta:
        ordering = ("-timestamp",)

    def email(self):
        return self.user.email

def get_settings():
    if ReminderSettings.objects.count() == 0:
        return None
    return ReminderSettings.objects.all()[0]

def get_upcoming_dates(now):
    settings = get_settings()
    if not settings or not settings.send_reminders or not settings.begin_date:
        raise StopIteration()

    if settings.get_interval() < 0:
        yield now, "Current newsletter"
        raise StopIteration()

    to_yield = 7
    current = settings.begin_date

    while to_yield > 0:
        if current >= now - datetime.timedelta(2 * settings.get_interval()):
            diff = current - now
            days = abs(diff.days) % 7
            weeks = abs(diff.days) / 7
            if diff.days > 0:
                if weeks == 0:
                    yield current, _("%(current)s (in %(days)s days)") % locals()
                else:
                    yield current, _("%(current)s (in %(weeks)s weeks)") % locals()
            else:
                if weeks == 0:
                    yield current, _("%(current)s (%(days)s days ago)") % locals()
                else:
                    yield current, _("%(current)s (%(weeks)s weeks ago)") % locals()
            to_yield -= 1
        current += datetime.timedelta(settings.get_interval())

def get_default_for_reminder(language):
    if NewsLetterTemplate.objects.language(language).filter(is_default_reminder=True).count() == 0:
        return None
    return NewsLetterTemplate.objects.language(language).filter(is_default_reminder=True)[0]

def get_default_for_newsitem(language):
    if NewsLetterTemplate.objects.language(language).filter(is_default_newsitem=True).count() == 0:
        return None
    return NewsLetterTemplate.objects.language(language).filter(is_default_newsitem=True)[0]

def get_prev_reminder_date(now, published=True):
    """Returns the date of the previous reminder or None if there's no
    such date"""

    settings = get_settings()

    if not settings or not settings.send_reminders or not settings.begin_date or now < settings.begin_date:
        return None

    if settings.interval == NO_INTERVAL:
        qs = NewsLetter.objects.filter(date__lte=now).exclude(date__gt=now).order_by("-date")
        if published:
            qs = qs.filter(published=published)
        qs = list(qs)
        if len(qs) == 0:
            return None
        return qs[0].date

    current = settings.begin_date
    prev = settings.begin_date

    while True:
        if current >= now:
            return prev
        prev = current
        current += datetime.timedelta(settings.get_interval())

def get_prev_reminder(now, published=True):
    """Returns the reminder (newsletter/tempate) to send at a given moment
    as a dict with languages as keys, or None if there is no such reminder"""
    def p(qs):
        if published:
            return qs.filter(published=published)
        return qs

    prev_date = get_prev_reminder_date(now, published)
    if prev_date is None:
        return None

    if p(NewsLetter.objects.filter(date=prev_date)).count():
        result = {}
        for language, name in settings.LANGUAGES:
            if p(NewsLetter.objects.language(language).filter(date=prev_date)):
                result[language] = p(NewsLetter.objects.language(language)).get(date=prev_date)
        return result

    result = {}
    for language, name in settings.LANGUAGES:
        template = get_default_for_reminder(language)
        if template is None:
            continue

        newsletter = MockNewsLetter()
        newsletter.date = prev_date
        newsletter.sender_email = template.sender_email
        newsletter.sender_name = template.sender_name
        newsletter.subject = template.subject
        newsletter.message = template.message
        result[language] = newsletter

    return result

def get_reminders_for_users(now, users):
    reminder_dict = get_prev_reminder(now)
    if not reminder_dict:
        raise StopIteration()

    batch_size = get_settings().batch_size if get_settings() else None

    yielded = 0
    for user in users:
        if batch_size and yielded >= batch_size:
            raise StopIteration

        info, _ = UserReminderInfo.objects.get_or_create(user=user, defaults={'active': True, 'last_reminder': user.date_joined})

        if not info.active:
            continue

        language = info.get_language()
        if not language in reminder_dict:
            language = settings.LANGUAGE_CODE

        reminder = reminder_dict[language]

        if info.last_reminder is None:
            yield user, reminder, language
            yielded += 1
            continue

        if get_settings() and get_settings().interval == WEEKLY_WITH_BATCHES:
            survey_users = SurveyUser.objects.filter(user=user, deleted=False)
            if not survey_users.count():
                survey_user = SurveyUser.objects.create(user=user, name=user.username)
                survey_users = SurveyUser.objects.filter(user=user)

            last_action_date = now - max(su.get_last_weekly_survey_date() for su in survey_users)
            if info.last_reminder and info.last_reminder > last_action_date:
                last_action_date = info.last_reminder

            last_action = last_action_date.days
            last_action_long_ago_enough = last_action >= 7

            if not last_action_long_ago_enough:
                continue

        if info.last_reminder < reminder.date:
            yield user, reminder, language
            yielded += 1

<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
import cv2

def augment_br_sat_hue_cont(image):

  def _augment_br_sat_hue_cont(x):
    # all functions include clamping for overflow values
    x = tf.image.random_brightness(x, max_delta=0.3)
    x = tf.image.random_saturation(x, lower=0.7, upper=1.3)
    #x = tf.image.random_hue(x, max_delta=0.032)
    #x = tf.image.random_contrast(x, lower=0.7, upper=1.3)
    return x

  with tf.name_scope('random_sat_hue_cont'):
    image = tf.map_fn(lambda x:_augment_br_sat_hue_cont(x), image)
  return image

def augment_gaussian_noise(images, std=0.2):
  with tf.name_scope('gaussian_noise'):
    noise = tf.random_normal(shape=tf.shape(images), mean=0.0, stddev=std, dtype=tf.float32)
  return images + noise

def augment_scale_translate_flip(images, img_size, boxes, flip, batch_size, scale_range=0.2):

  #batch_size = images.get_shape()[0]
  #batch_size = FLAGS.batch_size # this value should be fixed up

  # Translation
  with tf.name_scope('scale_trans'):
    scale = 1.0 + tf.random_uniform([1], minval=0.0, maxval=scale_range)
    size = tf.constant([img_size, img_size])
    new_size = scale*tf.cast(size, dtype=tf.float32)

    box_ind = tf.range(start=0, limit=batch_size, dtype=tf.int32)

    images = tf.image.crop_and_resize(
        images,
        boxes=boxes,
        box_ind=box_ind,
        crop_size=size
        )

  def flip_left_right(i):
    image = images[i]
    flip_or_not = flip[i]
    return tf.cond(flip_or_not, lambda: tf.reverse(image, axis=[1]), lambda: image)

  with tf.name_scope('flip'):
    idxs = tf.range(0, batch_size, dtype=tf.int32)
    images = tf.map_fn(lambda idx:flip_left_right(idx), idxs, dtype=tf.float32)
  return images

def cal_area(box):
  w, h = np.array(box[1]) - np.array(box[0])
  return w*h

def cal_iou(box1, box2):
  area1 = cal_area(box1)
  area2 = cal_area(box2)

  upleft = np.maximum(box1[0], box2[0])
  downright = np.minimum(box1[1], box2[1])

  intersect = np.maximum((.0, .0), downright - upleft)

  intersect_area = intersect[0]*intersect[1]
  union_area = area1 + area2 - intersect_area

  return float(intersect_area)/union_area

def cvt_bbox2cwh(bbox):
  upleft, downright = bbox
  x1, y1 = upleft
  x2, y2 = downright

  x, y = (x1 + x2)/2.0, (y1 + y2)/2.0
  w, h = (x2 - x1), (y2 - y1)

  return ((x, y), (w, h))

def cvt_cwh2bbox(cwh):
  xy, wh = cwh
  x, y = xy
  w, h = wh

  x1, y1 = x - w/2.0, y - h/2.0
  x2, y2 = x + w/2.0, y + h/2.0

  return ((x1, y1), (x2, y2))

def img_listup(imgs):
  size = len(imgs)
  (h, w) = imgs[0].shape[:2]

  total_w = 0
  for img in imgs:
    total_w += img.shape[1]
  out = np.zeros((h, total_w, 3), np.uint8)

  offset = 0
  for i in range(size):
    h, w = imgs[i].shape[:2]
    out[:h, offset: offset + w] = imgs[i]
    offset += w

  return out

def visualization_orig(img, _annot, idx2obj, palette):
  print("visualization_orig()")
  h, w = img.shape[:2]

  _w, _h = _annot[0, :2]
  scale_w, scale_h = float(w)/_w, float(h)/_h
  for box in _annot[1:]:
    idx, x1, x2, y1, y2 = box
    idx = int(idx)
    _color = palette[idx]
    color = (int(_color[2]), int(_color[1]), int(_color[0]))

    name = idx2obj[idx]

    cx, cy = (scale_w*(x1 + x2)/2.0, scale_h*(y1 + y2)/2.0)

    b = (int(scale_w*x1), int(scale_h*y1))
    e = (int(scale_w*x2), int(scale_h*y2))
    img = cv2.rectangle(img, b, e, color, 5)
    img = cv2.putText(img, name, b, cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255,255,255), 1)
    img = cv2.circle(img, (int(cx), int(cy)), 4, color, -1)

  return img

<commit_msg>update random-flip augmentation process<commit_after>import tensorflow as tf
import numpy as np
import cv2

def augment_br_sat_hue_cont(image):

  def _augment_br_sat_hue_cont(x):
    # all functions include clamping for overflow values
    x = tf.image.random_brightness(x, max_delta=0.3)
    x = tf.image.random_saturation(x, lower=0.7, upper=1.3)
    #x = tf.image.random_hue(x, max_delta=0.032)
    #x = tf.image.random_contrast(x, lower=0.7, upper=1.3)
    return x

  with tf.name_scope('random_sat_hue_cont'):
    image = tf.map_fn(lambda x:_augment_br_sat_hue_cont(x), image)
  return image

def augment_gaussian_noise(images, std=0.2):
  with tf.name_scope('gaussian_noise'):
    noise = tf.random_normal(shape=tf.shape(images), mean=0.0, stddev=std, dtype=tf.float32)
  return images + noise

def augment_scale_translate_flip(images, img_size, boxes, flip, batch_size, scale_range=0.2):

  #batch_size = images.get_shape()[0]
  #batch_size = FLAGS.batch_size # this value should be fixed up

  # Translation
  with tf.name_scope('scale_trans'):
    scale = 1.0 + tf.random_uniform([1], minval=0.0, maxval=scale_range)
    size = tf.constant([img_size, img_size])
    new_size = scale*tf.cast(size, dtype=tf.float32)

    box_ind = tf.range(start=0, limit=batch_size, dtype=tf.int32)

    images = tf.image.crop_and_resize(
        images,
        boxes=boxes,
        box_ind=box_ind,
        crop_size=size
        )

#  def flip_left_right(i):
#    image = images[i]
#    flip_or_not = flip[i]
#    return tf.cond(flip_or_not, lambda: tf.reverse(image, axis=[1]), lambda: image)
#
#  with tf.name_scope('flip'):
#    idxs = tf.range(0, batch_size, dtype=tf.int32)
#    images = tf.map_fn(lambda idx:flip_left_right(idx), idxs, dtype=tf.float32)

  with tf.name_scope('flip'):
    flipeds = tf.reverse(images, axis=[2])
    images = tf.where(flip, flipeds, images)
  return images

def cal_area(box):
  w, h = np.array(box[1]) - np.array(box[0])
  return w*h

def cal_iou(box1, box2):
  area1 = cal_area(box1)
  area2 = cal_area(box2)

  upleft = np.maximum(box1[0], box2[0])
  downright = np.minimum(box1[1], box2[1])

  intersect = np.maximum((.0, .0), downright - upleft)

  intersect_area = intersect[0]*intersect[1]
  union_area = area1 + area2 - intersect_area

  return float(intersect_area)/union_area

def cvt_bbox2cwh(bbox):
  upleft, downright = bbox
  x1, y1 = upleft
  x2, y2 = downright

  x, y = (x1 + x2)/2.0, (y1 + y2)/2.0
  w, h = (x2 - x1), (y2 - y1)

  return ((x, y), (w, h))

def cvt_cwh2bbox(cwh):
  xy, wh = cwh
  x, y = xy
  w, h = wh

  x1, y1 = x - w/2.0, y - h/2.0
  x2, y2 = x + w/2.0, y + h/2.0

  return ((x1, y1), (x2, y2))

def img_listup(imgs):
  size = len(imgs)
  (h, w) = imgs[0].shape[:2]

  total_w = 0
  for img in imgs:
    total_w += img.shape[1]
  out = np.zeros((h, total_w, 3), np.uint8)

  offset = 0
  for i in range(size):
    h, w = imgs[i].shape[:2]
    out[:h, offset: offset + w] = imgs[i]
    offset += w

  return out

def visualization_orig(img, _annot, idx2obj, palette):
  print("visualization_orig()")
  h, w = img.shape[:2]

  _w, _h = _annot[0, :2]
  scale_w, scale_h = float(w)/_w, float(h)/_h
  for box in _annot[1:]:
    idx, x1, x2, y1, y2 = box
    idx = int(idx)
    _color = palette[idx]
    color = (int(_color[2]), int(_color[1]), int(_color[0]))

    name = idx2obj[idx]

    cx, cy = (scale_w*(x1 + x2)/2.0, scale_h*(y1 + y2)/2.0)

    b = (int(scale_w*x1), int(scale_h*y1))
    e = (int(scale_w*x2), int(scale_h*y2))
    img = cv2.rectangle(img, b, e, color, 5)
    img = cv2.putText(img, name, b, cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255,255,255), 1)
    img = cv2.circle(img, (int(cx), int(cy)), 4, color, -1)

  return img

<|endoftext|>


<commit_before>import datetime
import os
import shutil
import time
import redis
from celery.task import Task
from utils import log as logging
from utils import s3_utils as s3
from django.conf import settings

class TaskFeeds(Task):
    name = 'task-feeds'

    def run(self, **kwargs):
        from apps.rss_feeds.models import Feed
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.utcnow()
        start = time.time()
        r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
        tasked_feeds_size = r.zcard('tasked_feeds')

        hour_ago = now - datetime.timedelta(hours=1)
        r.zremrangebyscore('fetched_feeds_last_hour', 0, int(hour_ago.strftime('%s')))

        now_timestamp = int(now.strftime("%s"))
        queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
        r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
        r.sadd('queued_feeds', *queued_feeds)
        logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
                        len(queued_feeds),
                        r.zcard('tasked_feeds'),
                        r.scard('queued_feeds'),
                        r.zcard('scheduled_updates')))

        # Regular feeds
        if tasked_feeds_size < 1000:
            feeds = r.srandmember('queued_feeds', 1000)
            Feed.task_feeds(feeds, verbose=True)
            active_count = len(feeds)
        else:
            logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
            active_count = 0
        cp1 = time.time()

        # Force refresh feeds
        refresh_feeds = Feed.objects.filter(
            active=True,
            fetched_once=False,
            active_subscribers__gte=1
        ).order_by('?')[:100]
        refresh_count = refresh_feeds.count()
        cp2 = time.time()

        # Mistakenly inactive feeds
        hours_ago = (now - datetime.timedelta(hours=1)).strftime('%s')
        old_tasked_feeds = r.zrangebyscore('tasked_feeds', 0, hours_ago)
        inactive_count = len(old_tasked_feeds)
        if inactive_count:
            r.zremrangebyscore('tasked_feeds', 0, hours_ago)
            r.sadd('queued_feeds', *old_tasked_feeds)
            logging.debug(" ---> ~SN~FBRe-queuing ~SB%s~SN dropped feeds (~SB%s/%s~SN queued/tasked)" % (
                            inactive_count,
                            r.scard('queued_feeds'),
                            r.zcard('tasked_feeds')))
        cp3 = time.time()

        old = now - datetime.timedelta(days=1)
        old_feeds = Feed.objects.filter(
            next_scheduled_update__lte=old,
            active_subscribers__gte=1
        ).order_by('?')[:500]
        old_count = old_feeds.count()
        cp4 = time.time()

        logging.debug(" ---> ~FBTasking ~SB~FC%s~SN~FB/~FC%s~FB (~FC%s~FB/~FC%s~SN~FB) feeds... (%.4s/%.4s/%.4s/%.4s)" % (
            active_count,
            refresh_count,
            inactive_count,
            old_count,
            cp1 - start,
            cp2 - cp1,
            cp3 - cp2,
            cp4 - cp3
        ))

        # Feed.task_feeds(feeds, verbose=False)
        Feed.task_feeds(refresh_feeds, verbose=False)
        Feed.task_feeds(old_feeds, verbose=False)

        logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
                        int((time.time() - start)),
                        r.zcard('tasked_feeds'),
                        r.scard('queued_feeds'),
                        r.zcard('scheduled_updates')))

class UpdateFeeds(Task):
    name = 'update-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_pks, **kwargs):
        from apps.rss_feeds.models import Feed
        from apps.statistics.models import MStatistics
        r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)

        mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
        compute_scores = bool(mongodb_replication_lag < 10)

        options = {
            'quick': float(MStatistics.get('quick_fetch', 0)),
            'compute_scores': compute_scores,
            'mongodb_replication_lag': mongodb_replication_lag,
        }

        if not isinstance(feed_pks, list):
            feed_pks = [feed_pks]

        for feed_pk in feed_pks:
            feed = Feed.get_by_id(feed_pk)
            if not feed or feed.pk != int(feed_pk):
                logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
                r.zrem('tasked_feeds', feed_pk)
            if feed:
                feed.update(**options)

class NewFeeds(Task):
    name = 'new-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_pks, **kwargs):
        from apps.rss_feeds.models import Feed
        if not isinstance(feed_pks, list):
            feed_pks = [feed_pks]

        options = {}
        for feed_pk in feed_pks:
            feed = Feed.get_by_id(feed_pk)
            feed.update(options=options)

class PushFeeds(Task):
    name = 'push-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_id, xml, **kwargs):
        from apps.rss_feeds.models import Feed
        from apps.statistics.models import MStatistics

        mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
        compute_scores = bool(mongodb_replication_lag < 60)

        options = {
            'feed_xml': xml,
            'compute_scores': compute_scores,
            'mongodb_replication_lag': mongodb_replication_lag,
        }
        feed = Feed.get_by_id(feed_id)
        feed.update(options=options)

class BackupMongo(Task):
    name = 'backup-mongo'
    max_retries = 0
    ignore_result = True

    def run(self, **kwargs):
        COLLECTIONS = "classifier_tag classifier_author classifier_feed classifier_title userstories starred_stories shared_stories category category_site sent_emails social_profile social_subscription social_services statistics feedback"

        date = time.strftime('%Y-%m-%d-%H-%M')
        collections = COLLECTIONS.split(' ')
        db_name = 'newsblur'
        dir_name = 'backup_mongo_%s' % date
        filename = '%s.tgz' % dir_name

        os.mkdir(dir_name)

        for collection in collections:
            cmd = 'mongodump  --db %s --collection %s -o %s' % (db_name, collection, dir_name)
            logging.debug(' ---> ~FMDumping ~SB%s~SN: %s' % (collection, cmd))
            os.system(cmd)

        cmd = 'tar -jcf %s %s' % (filename, dir_name)
        os.system(cmd)

        logging.debug(' ---> ~FRUploading ~SB~FM%s~SN~FR to S3...' % filename)
        s3.save_file_in_s3(filename)
        shutil.rmtree(dir_name)
        os.remove(filename)
        logging.debug(' ---> ~FRFinished uploading ~SB~FM%s~SN~FR to S3.' % filename)

class ScheduleImmediateFetches(Task):

    def run(self, feed_ids, **kwargs):
        from apps.rss_feeds.models import Feed

        if not isinstance(feed_ids, list):
            feed_ids = [feed_ids]

        Feed.schedule_feed_fetches_immediately(feed_ids)

class SchedulePremiumSetup(Task):

    def run(self, feed_ids, **kwargs):
        from apps.rss_feeds.models import Feed

        if not isinstance(feed_ids, list):
            feed_ids = [feed_ids]

        Feed.setup_feeds_for_premium_subscribers(feed_ids)

<commit_msg>Adding error_feeds queue to keep track of broken feeds.<commit_after>import datetime
import os
import shutil
import time
import redis
from celery.task import Task
from utils import log as logging
from utils import s3_utils as s3
from django.conf import settings

class TaskFeeds(Task):
    name = 'task-feeds'

    def run(self, **kwargs):
        from apps.rss_feeds.models import Feed
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.utcnow()
        start = time.time()
        r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
        tasked_feeds_size = r.zcard('tasked_feeds')

        hour_ago = now - datetime.timedelta(hours=1)
        r.zremrangebyscore('fetched_feeds_last_hour', 0, int(hour_ago.strftime('%s')))

        now_timestamp = int(now.strftime("%s"))
        queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
        r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
        r.sadd('queued_feeds', *queued_feeds)
        logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
                        len(queued_feeds),
                        r.zcard('tasked_feeds'),
                        r.scard('queued_feeds'),
                        r.zcard('scheduled_updates')))

        # Regular feeds
        if tasked_feeds_size < 1000:
            feeds = r.srandmember('queued_feeds', 1000)
            Feed.task_feeds(feeds, verbose=True)
            active_count = len(feeds)
        else:
            logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
            active_count = 0
        cp1 = time.time()

        # Force refresh feeds
        refresh_feeds = Feed.objects.filter(
            active=True,
            fetched_once=False,
            active_subscribers__gte=1
        ).order_by('?')[:100]
        refresh_count = refresh_feeds.count()
        cp2 = time.time()

        # Mistakenly inactive feeds
        hours_ago = (now - datetime.timedelta(hours=1)).strftime('%s')
        old_tasked_feeds = r.zrangebyscore('tasked_feeds', 0, hours_ago)
        inactive_count = len(old_tasked_feeds)
        if inactive_count:
            r.zremrangebyscore('tasked_feeds', 0, hours_ago)
            r.sadd('queued_feeds', *old_tasked_feeds)
            p = r.pipeline()
            for feed_id in old_tasked_feeds:
                p.zincrby('error_feeds', feed_id, 1)
            p.execute()
            logging.debug(" ---> ~SN~FBRe-queuing ~SB%s~SN dropped feeds (~SB%s/%s~SN queued/tasked)" % (
                            inactive_count,
                            r.scard('queued_feeds'),
                            r.zcard('tasked_feeds')))
        cp3 = time.time()

        old = now - datetime.timedelta(days=1)
        old_feeds = Feed.objects.filter(
            next_scheduled_update__lte=old,
            active_subscribers__gte=1
        ).order_by('?')[:500]
        old_count = old_feeds.count()
        cp4 = time.time()

        logging.debug(" ---> ~FBTasking ~SB~FC%s~SN~FB/~FC%s~FB (~FC%s~FB/~FC%s~SN~FB) feeds... (%.4s/%.4s/%.4s/%.4s)" % (
            active_count,
            refresh_count,
            inactive_count,
            old_count,
            cp1 - start,
            cp2 - cp1,
            cp3 - cp2,
            cp4 - cp3
        ))

        # Feed.task_feeds(feeds, verbose=False)
        Feed.task_feeds(refresh_feeds, verbose=False)
        Feed.task_feeds(old_feeds, verbose=False)

        logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
                        int((time.time() - start)),
                        r.zcard('tasked_feeds'),
                        r.scard('queued_feeds'),
                        r.zcard('scheduled_updates')))

class UpdateFeeds(Task):
    name = 'update-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_pks, **kwargs):
        from apps.rss_feeds.models import Feed
        from apps.statistics.models import MStatistics
        r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)

        mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
        compute_scores = bool(mongodb_replication_lag < 10)

        options = {
            'quick': float(MStatistics.get('quick_fetch', 0)),
            'compute_scores': compute_scores,
            'mongodb_replication_lag': mongodb_replication_lag,
        }

        if not isinstance(feed_pks, list):
            feed_pks = [feed_pks]

        for feed_pk in feed_pks:
            feed = Feed.get_by_id(feed_pk)
            if not feed or feed.pk != int(feed_pk):
                logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
                r.zrem('tasked_feeds', feed_pk)
            if feed:
                feed.update(**options)

class NewFeeds(Task):
    name = 'new-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_pks, **kwargs):
        from apps.rss_feeds.models import Feed
        if not isinstance(feed_pks, list):
            feed_pks = [feed_pks]

        options = {}
        for feed_pk in feed_pks:
            feed = Feed.get_by_id(feed_pk)
            feed.update(options=options)

class PushFeeds(Task):
    name = 'push-feeds'
    max_retries = 0
    ignore_result = True

    def run(self, feed_id, xml, **kwargs):
        from apps.rss_feeds.models import Feed
        from apps.statistics.models import MStatistics

        mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
        compute_scores = bool(mongodb_replication_lag < 60)

        options = {
            'feed_xml': xml,
            'compute_scores': compute_scores,
            'mongodb_replication_lag': mongodb_replication_lag,
        }
        feed = Feed.get_by_id(feed_id)
        feed.update(options=options)

class BackupMongo(Task):
    name = 'backup-mongo'
    max_retries = 0
    ignore_result = True

    def run(self, **kwargs):
        COLLECTIONS = "classifier_tag classifier_author classifier_feed classifier_title userstories starred_stories shared_stories category category_site sent_emails social_profile social_subscription social_services statistics feedback"

        date = time.strftime('%Y-%m-%d-%H-%M')
        collections = COLLECTIONS.split(' ')
        db_name = 'newsblur'
        dir_name = 'backup_mongo_%s' % date
        filename = '%s.tgz' % dir_name

        os.mkdir(dir_name)

        for collection in collections:
            cmd = 'mongodump  --db %s --collection %s -o %s' % (db_name, collection, dir_name)
            logging.debug(' ---> ~FMDumping ~SB%s~SN: %s' % (collection, cmd))
            os.system(cmd)

        cmd = 'tar -jcf %s %s' % (filename, dir_name)
        os.system(cmd)

        logging.debug(' ---> ~FRUploading ~SB~FM%s~SN~FR to S3...' % filename)
        s3.save_file_in_s3(filename)
        shutil.rmtree(dir_name)
        os.remove(filename)
        logging.debug(' ---> ~FRFinished uploading ~SB~FM%s~SN~FR to S3.' % filename)

class ScheduleImmediateFetches(Task):

    def run(self, feed_ids, **kwargs):
        from apps.rss_feeds.models import Feed

        if not isinstance(feed_ids, list):
            feed_ids = [feed_ids]

        Feed.schedule_feed_fetches_immediately(feed_ids)

class SchedulePremiumSetup(Task):

    def run(self, feed_ids, **kwargs):
        from apps.rss_feeds.models import Feed

        if not isinstance(feed_ids, list):
            feed_ids = [feed_ids]

        Feed.setup_feeds_for_premium_subscribers(feed_ids)

<|endoftext|>


<commit_before># coding=utf-8
from __future__ import absolute_import, division, print_function, \
  unicode_literals

import filters as f
from iota.commands import FilterCommand, RequestFilter
from iota.crypto.signing import KeyGenerator
from iota.crypto.types import Seed
from iota.filters import Trytes

__all__ = [
  'GetPrivateKeysCommand',
]

class GetPrivateKeysCommand(FilterCommand):
  """
  Implements `get_private_keys` multisig API command.

  References:
    - :py:meth:`iota.multisig.MultisigIota.get_private_key`
    - https://github.com/iotaledger/wiki/blob/master/multisigs.md
  """
  command = 'getPrivateKeys'

  def get_request_filter(self):
    return GetPrivateKeysRequestFilter()

  def get_response_filter(self):
    pass

  def _execute(self, request):
    count = request['count'] # type: Optional[int]
    index = request['index'] # type: int
    seed  = request['seed'] # type: Seed

    generator = KeyGenerator(seed)

    return {
      'keys': generator.get_keys(start=index, count=count),
    }

class GetPrivateKeysRequestFilter(RequestFilter):
  def __init__(self):
    super(GetPrivateKeysRequestFilter, self).__init__(
      {
        # ``count`` and ``index`` are optional.
        'count':  f.Type(int) | f.Min(1) | f.Optional(default=1),
        'index':  f.Type(int) | f.Min(0) | f.Optional(default=0),

        'seed':   f.Required | Trytes(result_type=Seed),
      },

      allow_missing_keys = {
        'count',
        'index',
      },
    )
<commit_msg>[#10] Fixed imports.<commit_after># coding=utf-8
from __future__ import absolute_import, division, print_function, \
  unicode_literals

import filters as f
from iota.commands import FilterCommand, RequestFilter
from iota.crypto.signing import KeyGenerator
from iota.crypto.types import Seed
from iota.filters import Trytes
from typing import Optional

__all__ = [
  'GetPrivateKeysCommand',
]

class GetPrivateKeysCommand(FilterCommand):
  """
  Implements `get_private_keys` multisig API command.

  References:
    - :py:meth:`iota.multisig.MultisigIota.get_private_key`
    - https://github.com/iotaledger/wiki/blob/master/multisigs.md
  """
  command = 'getPrivateKeys'

  def get_request_filter(self):
    return GetPrivateKeysRequestFilter()

  def get_response_filter(self):
    pass

  def _execute(self, request):
    count = request['count'] # type: Optional[int]
    index = request['index'] # type: int
    seed  = request['seed'] # type: Seed

    generator = KeyGenerator(seed)

    return {
      'keys': generator.get_keys(start=index, count=count),
    }

class GetPrivateKeysRequestFilter(RequestFilter):
  def __init__(self):
    super(GetPrivateKeysRequestFilter, self).__init__(
      {
        # ``count`` and ``index`` are optional.
        'count':  f.Type(int) | f.Min(1) | f.Optional(default=1),
        'index':  f.Type(int) | f.Min(0) | f.Optional(default=0),

        'seed':   f.Required | Trytes(result_type=Seed),
      },

      allow_missing_keys = {
        'count',
        'index',
      },
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Unit tests for standard_names.Collection
"""
from nose.tools import (assert_greater, assert_equal, assert_raises,
                        assert_in, assert_is_instance, assert_tuple_equal)

from standard_names import NamesRegistry, StandardName, BadNameError

def test_create_full():
    """Test creating default registry."""
    nreg = NamesRegistry()
    assert_greater(len(nreg), 0)

def test_create_empty():
    """Test creating default registry."""
    nreg = NamesRegistry(None)
    assert_equal(len(nreg), 0)

def test_bad_name():
    """Try to add an invalid name."""
    nreg = NamesRegistry(None)
    with assert_raises(BadNameError):
        nreg.add('air_temperature')

def test_add_string():
    """Add a string to the collection."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')

    assert_in(StandardName('air__temperature'), nreg)
    assert_in('air__temperature', nreg)
    assert_equal(len(nreg), 1)

    nreg.add('air__temperature')
    assert_equal(len(nreg), 1)

def test_collection_contains_names():
    """Make sure a Collection only contains StandardName objects."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    for name in nreg:
        assert_is_instance(name, StandardName)

def test_add_name():
    """Add a name to the collection."""
    nreg = NamesRegistry(None)
    nreg.add(StandardName('air__temperature'))

    assert_in(StandardName('air__temperature'), nreg)
    assert_in('air__temperature', nreg)

def test_unique_names():
    """List of unique names."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    assert_equal(len(nreg), 2)

    assert_in('air__temperature', nreg)
    assert_in('water__temperature', nreg)

def test_unique_objects():
    """List of unique objects."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    objs = nreg.objects

    assert_equal(len(objs), 2)
    assert_in('air', objs)
    assert_in('water', objs)

def test_unique_quantities():
    """List of unique quantities."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    quantities = nreg.quantities

    assert_tuple_equal(quantities, ('temperature', ))

def test_unique_operators():
    """List of unique operators."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')
    nreg.add('air__log_of_temperature')
    nreg.add('air__mean_of_log_of_temperature')

    operators = nreg.operators

    assert_equal(len(operators), 2)
    assert_in('log', operators)
    assert_in('mean', operators)
<commit_msg>Fixed test of __iter__.<commit_after>#!/usr/bin/env python
"""Unit tests for standard_names.Collection"""
import types

from nose.tools import (assert_greater, assert_equal, assert_raises,
                        assert_in, assert_is_instance, assert_tuple_equal)

from standard_names import NamesRegistry, StandardName, BadNameError

def test_create_full():
    """Test creating default registry."""
    nreg = NamesRegistry()
    assert_greater(len(nreg), 0)

def test_create_empty():
    """Test creating default registry."""
    nreg = NamesRegistry(None)
    assert_equal(len(nreg), 0)

def test_bad_name():
    """Try to add an invalid name."""
    nreg = NamesRegistry(None)
    with assert_raises(BadNameError):
        nreg.add('air_temperature')

def test_add_string():
    """Add a string to the collection."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')

    assert_in(StandardName('air__temperature'), nreg)
    assert_in('air__temperature', nreg)
    assert_equal(len(nreg), 1)

    nreg.add('air__temperature')
    assert_equal(len(nreg), 1)

def test_collection_contains_names():
    """Make sure a Collection only contains StandardName objects."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    for name in nreg:
        assert_is_instance(name, types.StringTypes)

def test_add_name():
    """Add a name to the collection."""
    nreg = NamesRegistry(None)
    nreg.add(StandardName('air__temperature'))

    assert_in(StandardName('air__temperature'), nreg)
    assert_in('air__temperature', nreg)

def test_unique_names():
    """List of unique names."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    assert_equal(len(nreg), 2)

    assert_in('air__temperature', nreg)
    assert_in('water__temperature', nreg)

def test_unique_objects():
    """List of unique objects."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    objs = nreg.objects

    assert_equal(len(objs), 2)
    assert_in('air', objs)
    assert_in('water', objs)

def test_unique_quantities():
    """List of unique quantities."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')

    quantities = nreg.quantities

    assert_tuple_equal(quantities, ('temperature', ))

def test_unique_operators():
    """List of unique operators."""
    nreg = NamesRegistry(None)
    nreg.add('air__temperature')
    nreg.add('water__temperature')
    nreg.add('air__log_of_temperature')
    nreg.add('air__mean_of_log_of_temperature')

    operators = nreg.operators

    assert_equal(len(operators), 2)
    assert_in('log', operators)
    assert_in('mean', operators)
<|endoftext|>