<commit_before>from django.conf import settings
from django.contrib.auth.models import User, Group
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import get_model
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from base.util import with_timestamp, with_author, remove_accents

# Competition-related models
@with_author
@with_timestamp
class Competition(models.Model):
    """
    Represents a competition. One roots site will usually hold several
    competitions, as there are usually several age categories or several
    subjects categories. Or both.
    """

    name = models.CharField(max_length=100,
                            verbose_name=_('competition name'),
                            unique=True)
    organizer_group = models.ForeignKey(Group,
                                        blank=True,
                                        null=True,
                                        verbose_name=_('organizer group'))

    # Fields added via foreign keys:

    #  competitionorgregistration_set
    #  competitionuserregistration_set
    #  gallery_set
    #  leaflet_set
    #  post_set
    #  problemset_set
    #  season_set
    #  user_set

    def __unicode__(self):
        return self.name

    def get_active_season(self):
        season_candidates = self.season_set.filter(start__lt=now(),
                                                   end__gt=now())
        if season_candidates.exists():
            return season_candidates[0]
        else:
            return None

    class Meta:
        ordering = ['name']
        verbose_name = _('competition')
        verbose_name_plural = _('competitions')

@with_author
@with_timestamp
class CompetitionUserRegistration(models.Model):
    """
    Represents a relation between user and competition. User himself can
    register into competition if he satisfies the conditions.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    user = models.ForeignKey('profiles.UserProfile',
                             verbose_name=_('user'))

    def __unicode__(self):
        return (self.user.__unicode__() + u" competes in " +
               self.competition.__unicode__())

    class Meta:
        ordering = ['added_at']
        verbose_name = _('user registration')
        verbose_name_plural = _('user registrations')

@with_author
@with_timestamp
class CompetitionOrgRegistration(models.Model):
    """
    Represents a relation between organizer and comeptition. Organizer can
    help organize multiple competitions. Organizer registrations have to
    be approved.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    organizer = models.ForeignKey('profiles.UserProfile',
                                  verbose_name=_('organizer'))
    approved = models.BooleanField(verbose_name=_('registration approved'))

    def __unicode__(self):
        return (self.organizer.user.__unicode__() + unicode(_(" organizes ")) +
               self.competition.__unicode__())

    class Meta:
        ordering = ['added_at']
        verbose_name = _('organizer registration')
        verbose_name_plural = _('organizer registration')

@with_author
@with_timestamp
class Season(models.Model):
    """
    Represents an one season of a competition. This is usually autumn or spring
    season. Using this model, however, we are not limited to 2 seasons per year.

    During each Season there might be several ProblemSets published as parts
    of that season.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    year = models.IntegerField(verbose_name=_('year'))
    number = models.IntegerField(verbose_name=_('number'))
    name = models.CharField(max_length=50, verbose_name=_('name'))
    join_deadline = models.DateTimeField(blank=True,
                                         null=True,
                                         verbose_name=_('join deadline'))
    start = models.DateTimeField(verbose_name=_('season start'))
    end = models.DateTimeField(verbose_name=_('season end'))

    def get_competitors(self):
        """
        Returns the list of the competitors in the given season as everybody
        who submitted at least one problem solution in that season.
        """

        competitors = User.objects.none()

        for series in self.series_set.all():
            competitors = competitors | series.get_competitors()

        return competitors

    def get_user_solutions(self, user):
        """
        Returns the list of lists of solutions of problems in this season
        for a particular user, each series in a separate list.

        If the problem was not submitted, None is used instead of a
        UserSolution object.
        """

        return [series.get_user_solutions()
                for series in self.series_set.all()]

    def get_user_solutions_with_total(self, user):
        """
        Returns a list of lists of user solutions with total score computed.

        Also gives a total score per season for this user. A total score may
        be computed using settings.ROOTS_TOTAL_SEASON_SCORE_FUNC. Fallbacks
        to a simple sum of series subtotals if not specified.
        """

        all_solutions = []
        total = 0

        for series in self.series_set.all():
            solutions, subtotal = series.get_user_solutions_with_total(user)
            all_solutions.append(solutions)
            total += subtotal

        custom_total_func = getattr(settings,
                                    'ROOTS_TOTAL_SEASON_SCORE_FUNC',
                                    None)

        if custom_total_func is not None:
            assert callable(custom_total_func)
            return custom_total_func(user, solutions)
        else:
            # Fallback to simple sum
            return all_solutions, total

    def get_series_nearest_deadline(self):
        """
        Returns the most relevant series for the deadline. That is usually the
        series which is still active, but closest to its deadline.

        If all series are past their deadline, returns the last series.
        If there are no series in this season, returns None.
        """

        active_series = self.series_set.filter(submission_deadline__gt=now())

        if active_series.exists():
            return active_series[0]
        else:
            if self.series_set.exists():
                return self.series_set.order_by('-submission_deadline')[0]
            else:
                return None

    def __unicode__(self):
        template = "{name} ({competition} {year}-{number})"
        return template.format(competition=remove_accents(self.competition),
                               year=self.year,
                               number=self.number,
                               name=remove_accents(self.name),
                               )

    class Meta:
        ordering = ['competition', 'year', 'number']
        verbose_name = _('Season')
        verbose_name_plural = _('Seasons')

@with_author
@with_timestamp
class Series(models.Model):
    """
    Represents one series of problems in the season of the competetion.
    """

    season = models.ForeignKey('competitions.Season',
                               verbose_name=_('season'))
    name = models.CharField(max_length=50,
                            verbose_name=_('name'))
    number = models.PositiveSmallIntegerField(verbose_name=_('number'))
    problemset = models.OneToOneField('problems.ProblemSet',
                                      blank=True,
                                      null=True,
                                      verbose_name=_('problem set assigned'))
    submission_deadline = models.DateTimeField(
                              verbose_name=_('series submission deadline'))
    is_active = models.BooleanField(default=False,
                                    verbose_name=_('is series active'))

    def get_competitors(self):
        """
        Returns the list of the competitors in the given series as everybody
        who submitted at least one problem solution.
        """

        competitors = User.objects.none()

        for problem in self.problemset.problems.all():
            problemset_competitors = User.objects.filter(
                usersolution__pk__in=problem.usersolution_set.all())
            competitors = competitors | problemset_competitors

        return competitors.distinct()

    def get_user_solutions(self, user):
        """
        Returns the list of solutions of problems in this series for a
        particular user. If the problem was not submitted, None is used
        instead of a UserSolution object.
        """

        solutions = []
        UserSolution = get_model('problems', 'UserSolution')

        # TODO: See if this can be done with a simple JOIN without making
        # multiple queries here. Not optimizing prematurely now though.

        for problem in self.problemset.problems.all():
            try:
                solution = UserSolution.objects.get(user=user, problem=problem)
                solutions.append(solution)
            except UserSolution.DoesNotExist:
                solutions.append(None)

        return solutions

    def get_user_solutions_with_total(self, user):
        """
        Returns a list of user solutions with total score computed.

        The computation of the total score can follow any arbitrary rules,
        it needs to be specified by the settings.ROOTS_SERIES_TOTAL_SCORE_FUNC.

        This defaults to simple sum.
        """

        solutions = self.get_user_solutions(user)

        custom_total_func = getattr(settings,
                                    'ROOTS_SERIES_TOTAL_SCORE_FUNC',
                                    None)

        if custom_total_func is not None:
            assert callable(custom_total_func)
            return custom_total_func(user, solutions)
        else:
            # Fallback to simple sum
            total_sum = sum([s.score or 0 for s in solutions if s is not None])
            return solutions, total_sum

    def get_results(self):
        results = []

        for competitor in self.get_competitors():
            solutions, total = self.get_user_solutions_with_total(competitor)
            results.append((competitor, solutions, total))

        # Sort the results by total_score
        results.sort(key=lambda x: x[2], reverse=True)

        return results

    def get_user_ranking(self, user):
        results = self.get_results()

        for i in range(0, len(results)):
            competitor, solutions, total = results[i]
            if user == competitor:
                return i + 1

        return None

    def is_past_submission_deadline(self):
        return now() > self.submission_deadline

    def is_nearest_deadline(self):
        # Series are returned sorted by the submission deadline
        return self == self.season.get_series_nearest_deadline()

    def clean(self, *args, **kwargs):
        if self.is_active:
            if not self.submission_deadline:
                raise ValidationError("Submission deadline must be set to "
                                      "make the series active")
            if not self.problemset:
                raise ValidationError("Corresponding set of problems must be "
                                      "set to make the series active")

            if self.is_past_submission_deadline():
                raise ValidationError("Series that is past its submission "
                                      "deadline cannot be made active")

        super(Series, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.name

    class Meta:

        ordering = ['submission_deadline']
        unique_together = ('season', 'number')
        verbose_name = _('Series')
        verbose_name_plural = _('Series')
<commit_msg>competitions: Move common user result handling to SeasonSeriesBaseMixin<commit_after>from django.conf import settings
from django.contrib.auth.models import User, Group
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import get_model
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from base.util import with_timestamp, with_author, remove_accents

# Competition-related models
@with_author
@with_timestamp
class Competition(models.Model):
    """
    Represents a competition. One roots site will usually hold several
    competitions, as there are usually several age categories or several
    subjects categories. Or both.
    """

    name = models.CharField(max_length=100,
                            verbose_name=_('competition name'),
                            unique=True)
    organizer_group = models.ForeignKey(Group,
                                        blank=True,
                                        null=True,
                                        verbose_name=_('organizer group'))

    # Fields added via foreign keys:

    #  competitionorgregistration_set
    #  competitionuserregistration_set
    #  gallery_set
    #  leaflet_set
    #  post_set
    #  problemset_set
    #  season_set
    #  user_set

    def __unicode__(self):
        return self.name

    def get_active_season(self):
        season_candidates = self.season_set.filter(start__lt=now(),
                                                   end__gt=now())
        if season_candidates.exists():
            return season_candidates[0]
        else:
            return None

    class Meta:
        ordering = ['name']
        verbose_name = _('competition')
        verbose_name_plural = _('competitions')

@with_author
@with_timestamp
class CompetitionUserRegistration(models.Model):
    """
    Represents a relation between user and competition. User himself can
    register into competition if he satisfies the conditions.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    user = models.ForeignKey('profiles.UserProfile',
                             verbose_name=_('user'))

    def __unicode__(self):
        return (self.user.__unicode__() + u" competes in " +
               self.competition.__unicode__())

    class Meta:
        ordering = ['added_at']
        verbose_name = _('user registration')
        verbose_name_plural = _('user registrations')

@with_author
@with_timestamp
class CompetitionOrgRegistration(models.Model):
    """
    Represents a relation between organizer and comeptition. Organizer can
    help organize multiple competitions. Organizer registrations have to
    be approved.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    organizer = models.ForeignKey('profiles.UserProfile',
                                  verbose_name=_('organizer'))
    approved = models.BooleanField(verbose_name=_('registration approved'))

    def __unicode__(self):
        return (self.organizer.user.__unicode__() + unicode(_(" organizes ")) +
               self.competition.__unicode__())

    class Meta:
        ordering = ['added_at']
        verbose_name = _('organizer registration')
        verbose_name_plural = _('organizer registration')

class SeasonSeriesBaseMixin(object):

    def get_results(self):
        results = []

        for competitor in self.get_competitors():
            solutions, total = self.get_user_solutions_with_total(competitor)
            results.append((competitor, solutions, total))

        # Sort the results by total_score
        results.sort(key=lambda x: x[2], reverse=True)

        return results

    def get_user_ranking(self, user):
        """
        Returns the user's ranking in the current series. Since user can be
        ranked on multiple places, e.g. if 3 users achieved the perfect socre,
        the ranking is represented by a tuple (which would be (1,3) in this
        case). If only one user achiveed perfect score, the resulting ranking
        would be (1,1).
        """

        results = self.get_results()
        start = None

        for i in range(0, len(results)):
            competitor, solutions, total = results[i]
            if user == competitor:
                start = end = i + 1

            # This depends on the fact that the results list is sorted
            # decreasingly by the total score
            elif total == start:
                end = i + 1

        return (start, end)

    def get_user_by_ranking(self, rank):
        """
        Returns the set of users at the given rank (which can be empty).
        """

        results = self.get_results()

        # Convert rank to list index
        index = rank - 1

        users_at_rank = set()
        first_users_score = None

        if index < len(results):
            for competitor, solutions, total in results[index:]:

                # Mark the first user's score
                if first_users_score is None:
                    first_users_score = total

                # We add each user that has the same score until we reach
                # a lower score, at which point we break
                if total == first_users_score:
                    users_at_rank.add(competitor)
                else:
                    break

        return users_at_rank

@with_author
@with_timestamp
class Season(models.Model, SeasonSeriesBaseMixin):
    """
    Represents an one season of a competition. This is usually autumn or spring
    season. Using this model, however, we are not limited to 2 seasons per year.

    During each Season there might be several ProblemSets published as parts
    of that season.
    """

    competition = models.ForeignKey('competitions.Competition',
                                    verbose_name=_('competition'))
    year = models.IntegerField(verbose_name=_('year'))
    number = models.IntegerField(verbose_name=_('number'))
    name = models.CharField(max_length=50, verbose_name=_('name'))
    join_deadline = models.DateTimeField(blank=True,
                                         null=True,
                                         verbose_name=_('join deadline'))
    start = models.DateTimeField(verbose_name=_('season start'))
    end = models.DateTimeField(verbose_name=_('season end'))

    def get_competitors(self):
        """
        Returns the list of the competitors in the given season as everybody
        who submitted at least one problem solution in that season.
        """

        competitors = User.objects.none()

        for series in self.series_set.all():
            competitors = competitors | series.get_competitors()

        return competitors

    def get_user_solutions(self, user):
        """
        Returns the list of lists of solutions of problems in this season
        for a particular user, each series in a separate list.

        If the problem was not submitted, None is used instead of a
        UserSolution object.
        """

        return [series.get_user_solutions()
                for series in self.series_set.all()]

    def get_user_solutions_with_total(self, user):
        """
        Returns a list of lists of user solutions with total score computed.

        Also gives a total score per season for this user. A total score may
        be computed using settings.ROOTS_TOTAL_SEASON_SCORE_FUNC. Fallbacks
        to a simple sum of series subtotals if not specified.
        """

        all_solutions = []
        total = 0

        for series in self.series_set.all():
            solutions, subtotal = series.get_user_solutions_with_total(user)
            all_solutions.append(solutions)
            total += subtotal

        custom_total_func = getattr(settings,
                                    'ROOTS_TOTAL_SEASON_SCORE_FUNC',
                                    None)

        if custom_total_func is not None:
            assert callable(custom_total_func)
            return custom_total_func(user, solutions)
        else:
            # Fallback to simple sum
            return all_solutions, total

    def get_series_nearest_deadline(self):
        """
        Returns the most relevant series for the deadline. That is usually the
        series which is still active, but closest to its deadline.

        If all series are past their deadline, returns the last series.
        If there are no series in this season, returns None.
        """

        active_series = self.series_set.filter(submission_deadline__gt=now())

        if active_series.exists():
            return active_series[0]
        else:
            if self.series_set.exists():
                return self.series_set.order_by('-submission_deadline')[0]
            else:
                return None

    def __unicode__(self):
        template = "{name} ({competition} {year}-{number})"
        return template.format(competition=remove_accents(self.competition),
                               year=self.year,
                               number=self.number,
                               name=remove_accents(self.name),
                               )

    class Meta:
        ordering = ['competition', 'year', 'number']
        verbose_name = _('Season')
        verbose_name_plural = _('Seasons')

@with_author
@with_timestamp
class Series(models.Model, SeasonSeriesBaseMixin):
    """
    Represents one series of problems in the season of the competetion.
    """

    season = models.ForeignKey('competitions.Season',
                               verbose_name=_('season'))
    name = models.CharField(max_length=50,
                            verbose_name=_('name'))
    number = models.PositiveSmallIntegerField(verbose_name=_('number'))
    problemset = models.OneToOneField('problems.ProblemSet',
                                      blank=True,
                                      null=True,
                                      verbose_name=_('problem set assigned'))
    submission_deadline = models.DateTimeField(
                              verbose_name=_('series submission deadline'))
    is_active = models.BooleanField(default=False,
                                    verbose_name=_('is series active'))

    def get_competitors(self):
        """
        Returns the list of the competitors in the given series as everybody
        who submitted at least one problem solution.
        """

        competitors = User.objects.none()

        for problem in self.problemset.problems.all():
            problemset_competitors = User.objects.filter(
                usersolution__pk__in=problem.usersolution_set.all())
            competitors = competitors | problemset_competitors

        return competitors.distinct()

    def get_user_solutions(self, user):
        """
        Returns the list of solutions of problems in this series for a
        particular user. If the problem was not submitted, None is used
        instead of a UserSolution object.
        """

        solutions = []
        UserSolution = get_model('problems', 'UserSolution')

        # TODO: See if this can be done with a simple JOIN without making
        # multiple queries here. Not optimizing prematurely now though.

        for problem in self.problemset.problems.all():
            try:
                solution = UserSolution.objects.get(user=user, problem=problem)
                solutions.append(solution)
            except UserSolution.DoesNotExist:
                solutions.append(None)

        return solutions

    def get_user_solutions_with_total(self, user):
        """
        Returns a list of user solutions with total score computed.

        The computation of the total score can follow any arbitrary rules,
        it needs to be specified by the settings.ROOTS_SERIES_TOTAL_SCORE_FUNC.

        This defaults to simple sum.
        """

        solutions = self.get_user_solutions(user)

        custom_total_func = getattr(settings,
                                    'ROOTS_SERIES_TOTAL_SCORE_FUNC',
                                    None)

        if custom_total_func is not None:
            assert callable(custom_total_func)
            return custom_total_func(user, solutions)
        else:
            # Fallback to simple sum
            total_sum = sum([s.score or 0 for s in solutions if s is not None])
            return solutions, total_sum

    def is_past_submission_deadline(self):
        return now() > self.submission_deadline

    def is_nearest_deadline(self):
        # Series are returned sorted by the submission deadline
        return self == self.season.get_series_nearest_deadline()

    def clean(self, *args, **kwargs):
        if self.is_active:
            if not self.submission_deadline:
                raise ValidationError("Submission deadline must be set to "
                                      "make the series active")
            if not self.problemset:
                raise ValidationError("Corresponding set of problems must be "
                                      "set to make the series active")

            if self.is_past_submission_deadline():
                raise ValidationError("Series that is past its submission "
                                      "deadline cannot be made active")

        super(Series, self).save(*args, **kwargs)

    def __unicode__(self):
        return self.name

    class Meta:

        ordering = ['submission_deadline']
        unique_together = ('season', 'number')
        verbose_name = _('Series')
        verbose_name_plural = _('Series')
<|endoftext|>


<commit_before># Copyright 2015 Vinicius Chiele. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""APScheduler implementation."""

import functools
import logging
import socket
import warnings

from apscheduler.events import EVENT_ALL
from apscheduler.schedulers.background import BackgroundScheduler
from flask import make_response
from . import api
from .utils import fix_job_def, pop_trigger

LOGGER = logging.getLogger('flask_apscheduler')

class APScheduler(object):
    """Provides a scheduler integrated to Flask."""

    def __init__(self, scheduler=None, app=None):
        self._scheduler = scheduler or BackgroundScheduler()
        self._host_name = socket.gethostname().lower()
        self._authentication_callback = None

        self.allowed_hosts = ['*']
        self.auth = None
        self.api_enabled = False
        self.api_prefix = '/scheduler'
        self.endpoint_prefix = 'scheduler.'
        self.app = None

        if app:
            self.init_app(app)

    @property
    def host_name(self):
        """Get the host name."""
        return self._host_name

    @property
    def running(self):
        """Get true whether the scheduler is running."""
        return self._scheduler.running

    @property
    def state(self):
        """Get the state of the scheduler."""
        return self._scheduler.state

    @property
    def scheduler(self):
        """Get the base scheduler."""
        return self._scheduler

    def init_app(self, app):
        """Initialize the APScheduler with a Flask application instance."""

        self.app = app
        self.app.apscheduler = self

        self._load_config()
        self._load_jobs()

        if self.api_enabled:
            self._load_api()

    def start(self, paused=False):
        """
        Start the scheduler.
        :param bool paused: if True, don't start job processing until resume is called.
        """
        if self.host_name not in self.allowed_hosts and '*' not in self.allowed_hosts:
            LOGGER.debug('Host name %s is not allowed to start the APScheduler. Servers allowed: %s' %
                         (self.host_name, ','.join(self.allowed_hosts)))
            return

        self._scheduler.start(paused=paused)

    def shutdown(self, wait=True):
        """
        Shut down the scheduler. Does not interrupt any currently running jobs.

        :param bool wait: ``True`` to wait until all currently executing jobs have finished
        :raises SchedulerNotRunningError: if the scheduler has not been started yet
        """

        self._scheduler.shutdown(wait)

    def pause(self):
        """
        Pause job processing in the scheduler.

        This will prevent the scheduler from waking up to do job processing until :meth:`resume`
        is called. It will not however stop any already running job processing.
        """
        self._scheduler.pause()

    def resume(self):
        """
        Resume job processing in the scheduler.
        """
        self._scheduler.resume()

    def add_listener(self, callback, mask=EVENT_ALL):
        """
        Add a listener for scheduler events.

        When a matching event  occurs, ``callback`` is executed with the event object as its
        sole argument. If the ``mask`` parameter is not provided, the callback will receive events
        of all types.

        For further info: https://apscheduler.readthedocs.io/en/latest/userguide.html#scheduler-events

        :param callback: any callable that takes one argument
        :param int mask: bitmask that indicates which events should be listened to
        """
        self._scheduler.add_listener(callback, mask)

    def remove_listener(self, callback):
        """
        Remove a previously added event listener.
        """
        self._scheduler.remove_listener(callback)

    def add_job(self, id, func, **kwargs):
        """
        Add the given job to the job list and wakes up the scheduler if it's already running.

        :param str id: explicit identifier for the job (for modifying it later)
        :param func: callable (or a textual reference to one) to run at the given time
        """

        job_def = dict(kwargs)
        job_def['id'] = id
        job_def['func'] = func
        job_def['name'] = job_def.get('name') or id

        fix_job_def(job_def)

        return self._scheduler.add_job(**job_def)

    def delete_job(self, id, jobstore=None):
        """
        DEPRECATED, use remove_job instead.

        Remove a job, preventing it from being run any more.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        warnings.warn('delete_job has been deprecated, use remove_job instead.', DeprecationWarning)

        self.remove_job(id, jobstore)

    def delete_all_jobs(self, jobstore=None):
        """
        DEPRECATED, use remove_all_jobs instead.

        Remove all jobs from the specified job store, or all job stores if none is given.

        :param str|unicode jobstore: alias of the job store
        """

        warnings.warn('delete_all_jobs has been deprecated, use remove_all_jobs instead.', DeprecationWarning)

        self.remove_all_jobs(jobstore)

    def remove_job(self, id, jobstore=None):
        """
        Remove a job, preventing it from being run any more.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """

        self._scheduler.remove_job(id, jobstore)

    def remove_all_jobs(self, jobstore=None):
        """
        Remove all jobs from the specified job store, or all job stores if none is given.

        :param str|unicode jobstore: alias of the job store
        """

        self._scheduler.remove_all_jobs(jobstore)

    def get_job(self, id, jobstore=None):
        """
        Return the Job that matches the given ``id``.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that most likely contains the job
        :return: the Job by the given ID, or ``None`` if it wasn't found
        :rtype: Job
        """

        return self._scheduler.get_job(id, jobstore)

    def get_jobs(self, jobstore=None):
        """
        Return a list of pending jobs (if the scheduler hasn't been started yet) and scheduled jobs, either from a
        specific job store or from all of them.

        :param str jobstore: alias of the job store
        :rtype: list[Job]
        """

        return self._scheduler.get_jobs(jobstore)

    def modify_job(self, id, jobstore=None, **changes):
        """
        Modify the properties of a single job. Modifications are passed to this method as extra keyword arguments.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """

        fix_job_def(changes)

        if 'trigger' in changes:
            trigger, trigger_args = pop_trigger(changes)
            self._scheduler.reschedule_job(id, jobstore, trigger, **trigger_args)

        return self._scheduler.modify_job(id, jobstore, **changes)

    def pause_job(self, id, jobstore=None):
        """
        Pause the given job until it is explicitly resumed.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        self._scheduler.pause_job(id, jobstore)

    def resume_job(self, id, jobstore=None):
        """
        Resume the schedule of the given job, or removes the job if its schedule is finished.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        self._scheduler.resume_job(id, jobstore)

    def run_job(self, id, jobstore=None):
        """
        Run the given job without scheduling it.
        :param id: the identifier of the job.
        :param str jobstore: alias of the job store that contains the job
        :return:
        """
        job = self._scheduler.get_job(id, jobstore)

        if not job:
            raise LookupError(id)

        job.func(*job.args, **job.kwargs)

    def authenticate(self, func):
        """
        A decorator that is used to register a function to authenticate a user.
        :param func: The callback to authenticate.
        """
        self._authentication_callback = func
        return func

    def _load_config(self):
        """
        Load the configuration from the Flask configuration.
        """
        options = dict()

        job_stores = self.app.config.get('SCHEDULER_JOBSTORES')
        if job_stores:
            options['jobstores'] = job_stores

        executors = self.app.config.get('SCHEDULER_EXECUTORS')
        if executors:
            options['executors'] = executors

        job_defaults = self.app.config.get('SCHEDULER_JOB_DEFAULTS')
        if job_defaults:
            options['job_defaults'] = job_defaults

        timezone = self.app.config.get('SCHEDULER_TIMEZONE')
        if timezone:
            options['timezone'] = timezone

        self._scheduler.configure(**options)

        self.auth = self.app.config.get('SCHEDULER_AUTH', self.auth)
        self.api_enabled = self.app.config.get('SCHEDULER_VIEWS_ENABLED', self.api_enabled)  # for compatibility reason
        self.api_enabled = self.app.config.get('SCHEDULER_API_ENABLED', self.api_enabled)
        self.api_prefix = self.app.config.get('SCHEDULER_API_PREFIX', self.api_prefix)
        self.endpoint_prefix = self.app.config.get('SCHEDULER_ENDPOINT_PREFIX', self.endpoint_prefix)
        self.allowed_hosts = self.app.config.get('SCHEDULER_ALLOWED_HOSTS', self.allowed_hosts)

    def _load_jobs(self):
        """
        Load the job definitions from the Flask configuration.
        """
        jobs = self.app.config.get('SCHEDULER_JOBS')

        if not jobs:
            jobs = self.app.config.get('JOBS')

        if jobs:
            for job in jobs:
                self.add_job(**job)

    def _load_api(self):
        """
        Add the routes for the scheduler API.
        """
        self.app.add_url_rule(
            self.api_prefix + '',
            self.endpoint_prefix + 'get_scheduler_info',
            self._apply_auth(api.get_scheduler_info)
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs',
            self.endpoint_prefix + 'add_job',
            self._apply_auth(api.add_job),
            methods=['POST']
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs',
            self.endpoint_prefix + 'get_jobs',
            self._apply_auth(api.get_jobs)
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>',
            self.endpoint_prefix + 'get_job',
            self._apply_auth(api.get_job)
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>',
            self.endpoint_prefix + 'delete_job',
            self._apply_auth(api.delete_job),
            methods=['DELETE']
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>',
            self.endpoint_prefix + 'update_job',
            self._apply_auth(api.update_job),
            methods=['PATCH']
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>/pause',
            self.endpoint_prefix + 'pause_job',
            self._apply_auth(api.pause_job),
            methods=['POST']
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>/resume',
            self.endpoint_prefix + 'resume_job',
            self._apply_auth(api.resume_job),
            methods=['POST']
        )
        self.app.add_url_rule(
            self.api_prefix + '/jobs/<job_id>/run',
            self.endpoint_prefix + 'run_job',
            self._apply_auth(api.run_job),
            methods=['POST']
        )

    def _apply_auth(self, view_func):
        """
        Apply decorator to authenticate the user who is making the request.
        :param view_func: The flask view func.
        """
        @functools.wraps(view_func)
        def decorated(*args, **kwargs):
            if not self.auth:
                return view_func(*args, **kwargs)

            auth_data = self.auth.get_authorization()

            if auth_data is None:
                return self._handle_authentication_error()

            if not self._authentication_callback or not self._authentication_callback(auth_data):
                return self._handle_authentication_error()

            return view_func(*args, **kwargs)

        return decorated

    def _handle_authentication_error(self):
        """
        Return an authentication error.
        """
        response = make_response('Access Denied')
        response.headers['WWW-Authenticate'] = self.auth.get_authenticate_header()
        response.status_code = 401
        return response
<commit_msg>Improve addition of flask endpoints<commit_after># Copyright 2015 Vinicius Chiele. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""APScheduler implementation."""

import functools
import logging
import socket
import warnings

from apscheduler.events import EVENT_ALL
from apscheduler.schedulers.background import BackgroundScheduler
from flask import make_response
from . import api
from .utils import fix_job_def, pop_trigger

LOGGER = logging.getLogger('flask_apscheduler')

class APScheduler(object):
    """Provides a scheduler integrated to Flask."""

    def __init__(self, scheduler=None, app=None):
        self._scheduler = scheduler or BackgroundScheduler()
        self._host_name = socket.gethostname().lower()
        self._authentication_callback = None

        self.allowed_hosts = ['*']
        self.auth = None
        self.api_enabled = False
        self.api_prefix = '/scheduler'
        self.endpoint_prefix = 'scheduler.'
        self.app = None

        if app:
            self.init_app(app)

    @property
    def host_name(self):
        """Get the host name."""
        return self._host_name

    @property
    def running(self):
        """Get true whether the scheduler is running."""
        return self._scheduler.running

    @property
    def state(self):
        """Get the state of the scheduler."""
        return self._scheduler.state

    @property
    def scheduler(self):
        """Get the base scheduler."""
        return self._scheduler

    def init_app(self, app):
        """Initialize the APScheduler with a Flask application instance."""

        self.app = app
        self.app.apscheduler = self

        self._load_config()
        self._load_jobs()

        if self.api_enabled:
            self._load_api()

    def start(self, paused=False):
        """
        Start the scheduler.
        :param bool paused: if True, don't start job processing until resume is called.
        """
        if self.host_name not in self.allowed_hosts and '*' not in self.allowed_hosts:
            LOGGER.debug('Host name %s is not allowed to start the APScheduler. Servers allowed: %s' %
                         (self.host_name, ','.join(self.allowed_hosts)))
            return

        self._scheduler.start(paused=paused)

    def shutdown(self, wait=True):
        """
        Shut down the scheduler. Does not interrupt any currently running jobs.

        :param bool wait: ``True`` to wait until all currently executing jobs have finished
        :raises SchedulerNotRunningError: if the scheduler has not been started yet
        """

        self._scheduler.shutdown(wait)

    def pause(self):
        """
        Pause job processing in the scheduler.

        This will prevent the scheduler from waking up to do job processing until :meth:`resume`
        is called. It will not however stop any already running job processing.
        """
        self._scheduler.pause()

    def resume(self):
        """
        Resume job processing in the scheduler.
        """
        self._scheduler.resume()

    def add_listener(self, callback, mask=EVENT_ALL):
        """
        Add a listener for scheduler events.

        When a matching event  occurs, ``callback`` is executed with the event object as its
        sole argument. If the ``mask`` parameter is not provided, the callback will receive events
        of all types.

        For further info: https://apscheduler.readthedocs.io/en/latest/userguide.html#scheduler-events

        :param callback: any callable that takes one argument
        :param int mask: bitmask that indicates which events should be listened to
        """
        self._scheduler.add_listener(callback, mask)

    def remove_listener(self, callback):
        """
        Remove a previously added event listener.
        """
        self._scheduler.remove_listener(callback)

    def add_job(self, id, func, **kwargs):
        """
        Add the given job to the job list and wakes up the scheduler if it's already running.

        :param str id: explicit identifier for the job (for modifying it later)
        :param func: callable (or a textual reference to one) to run at the given time
        """

        job_def = dict(kwargs)
        job_def['id'] = id
        job_def['func'] = func
        job_def['name'] = job_def.get('name') or id

        fix_job_def(job_def)

        return self._scheduler.add_job(**job_def)

    def delete_job(self, id, jobstore=None):
        """
        DEPRECATED, use remove_job instead.

        Remove a job, preventing it from being run any more.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        warnings.warn('delete_job has been deprecated, use remove_job instead.', DeprecationWarning)

        self.remove_job(id, jobstore)

    def delete_all_jobs(self, jobstore=None):
        """
        DEPRECATED, use remove_all_jobs instead.

        Remove all jobs from the specified job store, or all job stores if none is given.

        :param str|unicode jobstore: alias of the job store
        """

        warnings.warn('delete_all_jobs has been deprecated, use remove_all_jobs instead.', DeprecationWarning)

        self.remove_all_jobs(jobstore)

    def remove_job(self, id, jobstore=None):
        """
        Remove a job, preventing it from being run any more.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """

        self._scheduler.remove_job(id, jobstore)

    def remove_all_jobs(self, jobstore=None):
        """
        Remove all jobs from the specified job store, or all job stores if none is given.

        :param str|unicode jobstore: alias of the job store
        """

        self._scheduler.remove_all_jobs(jobstore)

    def get_job(self, id, jobstore=None):
        """
        Return the Job that matches the given ``id``.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that most likely contains the job
        :return: the Job by the given ID, or ``None`` if it wasn't found
        :rtype: Job
        """

        return self._scheduler.get_job(id, jobstore)

    def get_jobs(self, jobstore=None):
        """
        Return a list of pending jobs (if the scheduler hasn't been started yet) and scheduled jobs, either from a
        specific job store or from all of them.

        :param str jobstore: alias of the job store
        :rtype: list[Job]
        """

        return self._scheduler.get_jobs(jobstore)

    def modify_job(self, id, jobstore=None, **changes):
        """
        Modify the properties of a single job. Modifications are passed to this method as extra keyword arguments.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """

        fix_job_def(changes)

        if 'trigger' in changes:
            trigger, trigger_args = pop_trigger(changes)
            self._scheduler.reschedule_job(id, jobstore, trigger, **trigger_args)

        return self._scheduler.modify_job(id, jobstore, **changes)

    def pause_job(self, id, jobstore=None):
        """
        Pause the given job until it is explicitly resumed.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        self._scheduler.pause_job(id, jobstore)

    def resume_job(self, id, jobstore=None):
        """
        Resume the schedule of the given job, or removes the job if its schedule is finished.

        :param str id: the identifier of the job
        :param str jobstore: alias of the job store that contains the job
        """
        self._scheduler.resume_job(id, jobstore)

    def run_job(self, id, jobstore=None):
        """
        Run the given job without scheduling it.
        :param id: the identifier of the job.
        :param str jobstore: alias of the job store that contains the job
        :return:
        """
        job = self._scheduler.get_job(id, jobstore)

        if not job:
            raise LookupError(id)

        job.func(*job.args, **job.kwargs)

    def authenticate(self, func):
        """
        A decorator that is used to register a function to authenticate a user.
        :param func: The callback to authenticate.
        """
        self._authentication_callback = func
        return func

    def _load_config(self):
        """
        Load the configuration from the Flask configuration.
        """
        options = dict()

        job_stores = self.app.config.get('SCHEDULER_JOBSTORES')
        if job_stores:
            options['jobstores'] = job_stores

        executors = self.app.config.get('SCHEDULER_EXECUTORS')
        if executors:
            options['executors'] = executors

        job_defaults = self.app.config.get('SCHEDULER_JOB_DEFAULTS')
        if job_defaults:
            options['job_defaults'] = job_defaults

        timezone = self.app.config.get('SCHEDULER_TIMEZONE')
        if timezone:
            options['timezone'] = timezone

        self._scheduler.configure(**options)

        self.auth = self.app.config.get('SCHEDULER_AUTH', self.auth)
        self.api_enabled = self.app.config.get('SCHEDULER_VIEWS_ENABLED', self.api_enabled)  # for compatibility reason
        self.api_enabled = self.app.config.get('SCHEDULER_API_ENABLED', self.api_enabled)
        self.api_prefix = self.app.config.get('SCHEDULER_API_PREFIX', self.api_prefix)
        self.endpoint_prefix = self.app.config.get('SCHEDULER_ENDPOINT_PREFIX', self.endpoint_prefix)
        self.allowed_hosts = self.app.config.get('SCHEDULER_ALLOWED_HOSTS', self.allowed_hosts)

    def _load_jobs(self):
        """
        Load the job definitions from the Flask configuration.
        """
        jobs = self.app.config.get('SCHEDULER_JOBS')

        if not jobs:
            jobs = self.app.config.get('JOBS')

        if jobs:
            for job in jobs:
                self.add_job(**job)

    def _load_api(self):
        """
        Add the routes for the scheduler API.
        """
        self._add_url_route('get_scheduler_info', '', api.get_scheduler_info, 'GET')
        self._add_url_route('add_job', '/jobs', api.add_job, 'POST')
        self._add_url_route('get_job', '/jobs/<job_id>', api.get_job, 'GET')
        self._add_url_route('get_jobs', '/jobs', api.get_jobs, 'GET')
        self._add_url_route('delete_job', '/jobs/<job_id>', api.delete_job, 'DELETE')
        self._add_url_route('update_job', '/jobs/<job_id>', api.update_job, 'PATCH')
        self._add_url_route('pause_job', '/jobs/<job_id>/pause', api.pause_job, 'POST')
        self._add_url_route('resume_job', '/jobs/<job_id>/resume', api.resume_job, 'POST')
        self._add_url_route('run_job', '/jobs/<job_id>/run', api.run_job, 'POST')

    def _add_url_route(self, endpoint, rule, view_func, method):
        """
        Add a Flask route.
        :param str endpoint: The endpoint name.
        :param str rule: The endpoint url.
        :param view_func: The endpoint func
        :param str method: The http method.
        """
        if self.api_prefix:
            rule = self.api_prefix + rule

        if self.endpoint_prefix:
            endpoint = self.endpoint_prefix + endpoint

        self.app.add_url_rule(
            rule,
            endpoint,
            self._apply_auth(view_func),
            methods=[method]
        )

    def _apply_auth(self, view_func):
        """
        Apply decorator to authenticate the user who is making the request.
        :param view_func: The flask view func.
        """
        @functools.wraps(view_func)
        def decorated(*args, **kwargs):
            if not self.auth:
                return view_func(*args, **kwargs)

            auth_data = self.auth.get_authorization()

            if auth_data is None:
                return self._handle_authentication_error()

            if not self._authentication_callback or not self._authentication_callback(auth_data):
                return self._handle_authentication_error()

            return view_func(*args, **kwargs)

        return decorated

    def _handle_authentication_error(self):
        """
        Return an authentication error.
        """
        response = make_response('Access Denied')
        response.headers['WWW-Authenticate'] = self.auth.get_authenticate_header()
        response.status_code = 401
        return response
<|endoftext|>


<commit_before>from loguru import logger

from flexget import plugin
from flexget.config_schema import one_or_more
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.cached_input import cached
from flexget.utils.requests import RequestException

logger = logger.bind(name='kitsu')

class KitsuAnime:
    """
    Creates an entry for each item in your kitsu.io list.

    Syntax:

    kitsu:
      username: <value>
      lists:
        - <current|planned|completed|on_hold|dropped>
        - <current|planned|completed|on_hold|dropped>
      type:
        - <ona|ova|tv|movie|music|special>
        - <ona|ova|tv|movie|music|special>
      status: <airing|finished>
      latest: <yes|no>
    """

    schema = {
        'type': 'object',
        'properties': {
            'username': {'type': 'string'},
            'lists': one_or_more(
                {
                    'type': 'string',
                    'enum': ['current', 'planned', 'completed', 'on_hold', 'dropped'],
                }
            ),
            'type': one_or_more(
                {'type': 'string', 'enum': ['ona', 'ova', 'tv', 'movie', 'music', 'special']}
            ),
            'latest': {'type': 'boolean', 'default': False},
            'status': {'type': 'string', 'enum': ['airing', 'finished']},
        },
        'required': ['username'],
        'additionalProperties': False,
    }

    @cached('kitsu', persist='2 hours')
    def on_task_input(self, task, config):
        user_payload = {'filter[name]': config['username']}
        try:
            user_response = task.requests.get(
                'https://kitsu.io/api/edge/users', params=user_payload
            )
        except RequestException as e:
            error_message = 'Error finding User url: {url}'.format(url=e.request.url)
            if hasattr(e, 'response'):
                error_message += ' status: {status}'.format(status=e.response.status_code)
            logger.opt(exception=True).debug(error_message)
            raise plugin.PluginError(error_message)
        user = user_response.json()
        if not len(user['data']):
            raise plugin.PluginError(
                'no such username found "{name}"'.format(name=config['username'])
            )
        next_url = 'https://kitsu.io/api/edge/users/{id}/library-entries'.format(
            id=user['data'][0]['id']
        )
        payload = {
            'filter[status]': ','.join(config['lists']),
            'filter[media_type]': 'Anime',
            'include': 'media',
            'page[limit]': 20,
        }
        try:
            response = task.requests.get(next_url, params=payload)
        except RequestException as e:
            error_message = 'Error getting list from {url}'.format(url=e.request.url)
            if hasattr(e, 'response'):
                error_message += ' status: {status}'.format(status=e.response.status_code)
            logger.opt(exception=True).debug(error_message)
            raise plugin.PluginError(error_message)

        json_data = response.json()

        while json_data:

            for item, anime in zip(json_data['data'], json_data['included']):
                if item['relationships']['media']['data']['id'] != anime['id']:
                    raise ValueError(
                        'Anime IDs {id1} and {id2} do not match'.format(
                            id1=item['relationships']['media']['data']['id'], id2=anime['id']
                        )
                    )
                status = config.get('status')
                if status is not None:
                    if status == 'airing' and anime['attributes']['endDate'] is not None:
                        continue
                    if status == 'finished' and anime['attributes']['endDate'] is None:
                        continue

                types = config.get('type')
                if types is not None:
                    subType = anime['attributes']['subtype']
                    if subType is None or not subType.lower() in types:
                        continue

                entry = Entry()
                entry['title'] = anime['attributes']['canonicalTitle']
                titles_en = anime['attributes']['titles'].get('en')
                if titles_en:
                    entry['kitsu_title_en'] = titles_en
                titles_en_jp = anime['attributes']['titles'].get('en_jp')
                if titles_en_jp:
                    entry['kitsu_title_en_jp'] = titles_en_jp
                titles_ja_jp = anime['attributes']['titles'].get('ja_jp')
                if titles_ja_jp:
                    entry['kitsu_title_ja_jp'] = titles_ja_jp
                entry['url'] = anime['links']['self']
                if entry.isvalid():
                    if config.get('latest'):
                        entry['series_episode'] = item['progress']
                        entry['series_id_type'] = 'sequence'
                        entry['title'] += ' ' + str(entry['progress'])
                    yield entry

            next_url = json_data['links'].get('next')
            if next_url:
                try:
                    response = task.requests.get(next_url)
                except RequestException as e:
                    error_message = 'Error getting list from next page url: {url}'.format(
                        url=e.request.url
                    )
                    if hasattr(e, 'response'):
                        error_message += ' status: {status}'.format(status=e.response.status_code)
                    logger.opt(exception=True).debug(error_message)
                    raise plugin.PluginError(error_message)
                json_data = response.json()
            else:
                break

@event('plugin.register')
def register_plugin():
    plugin.register(KitsuAnime, 'kitsu', api_ver=2)
<commit_msg>Fix Kitsu.io API errors<commit_after>from loguru import logger

from flexget import plugin
from flexget.config_schema import one_or_more
from flexget.entry import Entry
from flexget.event import event
from flexget.utils.cached_input import cached
from flexget.utils.requests import RequestException

logger = logger.bind(name='kitsu')

class KitsuAnime:
    """
    Creates an entry for each item in your kitsu.io list.

    Syntax:

    kitsu:
      username: <value>
      user_id: <value>
      lists:
        - <current|planned|completed|on_hold|dropped>
        - <current|planned|completed|on_hold|dropped>
      type:
        - <ona|ova|tv|movie|music|special>
        - <ona|ova|tv|movie|music|special>
      status: <airing|finished>
      latest: <yes|no>
    """

    schema = {
        'type': 'object',
        'properties': {
            'username': {'type': 'string'},
            'user_id': {'type': 'string'},
            'lists': one_or_more(
                {
                    'type': 'string',
                    'enum': ['current', 'planned', 'completed', 'on_hold', 'dropped'],
                }
            ),
            'type': one_or_more(
                {'type': 'string', 'enum': ['ona', 'ova', 'tv', 'movie', 'music', 'special']}
            ),
            'latest': {'type': 'boolean', 'default': False},
            'status': {'type': 'string', 'enum': ['airing', 'finished']},
        },
        'oneOf': [
            {'required': ['username']},
            {'required': ['user_id']}
        ],
        'additionalProperties': False,
    }

    @cached('kitsu', persist='2 hours')
    def on_task_input(self, task, config):
        user_id = self._resolve_user_id(task, config)
        next_url = 'https://kitsu.io/api/edge/users/{user_id}/library-entries'.format(
            user_id=user_id
        )

        payload = {
            'filter[status]': ','.join(config['lists']),
            'filter[kind]': 'anime',
            'include': 'anime',
            'fields[anime]': 'canonicalTitle,titles,endDate,subtype',
            'fields[libraryEntries]': 'anime',
            'page[limit]': 20,
        }

        try:
            response = task.requests.get(next_url, params=payload)
        except RequestException as e:
            error_message = 'Error getting list from {url}'.format(url=e.request.url)
            if hasattr(e, 'response'):
                error_message += ' status: {status}'.format(status=e.response.status_code)
            logger.opt(exception=True).debug(error_message)
            raise plugin.PluginError(error_message)
        json_data = response.json()

        while json_data:
            anime_dict = {relation['id']: relation for relation in json_data['included'] if relation['type'] == 'anime'}

            for item in json_data['data']:
                if item['relationships']['anime']['data'] is None:
                    logger.opt(exception=True).debug('Anime relation missing')
                    continue

                anime = anime_dict[item['relationships']['anime']['data']['id']]

                status = config.get('status')
                if status is not None:
                    if status == 'airing' and anime['attributes']['endDate'] is not None:
                        continue
                    if status == 'finished' and anime['attributes']['endDate'] is None:
                        continue

                types = config.get('type')
                if types is not None:
                    subType = anime['attributes']['subtype']
                    if subType is None or not subType.lower() in types:
                        continue

                entry = Entry()
                entry['title'] = anime['attributes']['canonicalTitle']
                titles_en = anime['attributes']['titles'].get('en')
                if titles_en:
                    entry['kitsu_title_en'] = titles_en
                titles_en_jp = anime['attributes']['titles'].get('en_jp')
                if titles_en_jp:
                    entry['kitsu_title_en_jp'] = titles_en_jp
                titles_ja_jp = anime['attributes']['titles'].get('ja_jp')
                if titles_ja_jp:
                    entry['kitsu_title_ja_jp'] = titles_ja_jp
                entry['url'] = anime['links']['self']
                if entry.isvalid():
                    if config.get('latest'):
                        entry['series_episode'] = item['progress']
                        entry['series_id_type'] = 'sequence'
                        entry['title'] += ' ' + str(entry['progress'])
                    yield entry

            next_url = json_data['links'].get('next')
            if next_url:
                try:
                    response = task.requests.get(next_url)
                except RequestException as e:
                    error_message = 'Error getting list from next page url: {url}'.format(
                        url=e.request.url
                    )
                    if hasattr(e, 'response'):
                        error_message += ' status: {status}'.format(status=e.response.status_code)
                    logger.opt(exception=True).debug(error_message)
                    raise plugin.PluginError(error_message)
                json_data = response.json()
            else:
                break

    def _resolve_user_id(self, task, config):
        user_id = config.get('user_id')

        if user_id is None:
            user_payload = {'filter[name]': config['username']}
            try:
                user_response = task.requests.get(
                    'https://kitsu.io/api/edge/users', params=user_payload
                )
            except RequestException as e:
                error_message = 'Error finding User url: {url}'.format(url=e.request.url)
                if hasattr(e, 'response'):
                    error_message += ' status: {status}'.format(status=e.response.status_code)
                logger.opt(exception=True).debug(error_message)
                raise plugin.PluginError(error_message)
            user = user_response.json()
            if not len(user['data']):
                raise plugin.PluginError(
                    'no such username found "{name}"'.format(name=config['username'])
                )
            user_id = user['data'][0]['id']

        return user_id

@event('plugin.register')
def register_plugin():
    plugin.register(KitsuAnime, 'kitsu', api_ver=2)
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
from collections import MutableMapping
import logging
import subprocess

from flexget import plugin
from flexget.event import event
from flexget.config_schema import one_or_more
from flexget.utils.template import render_from_entry, render_from_task, RenderError
from flexget.utils.tools import io_encoding

log = logging.getLogger('exec')

class EscapingDict(MutableMapping):
    """Helper class, same as a dict, but returns all string value with quotes escaped."""

    def __init__(self, mapping):
        self._data = mapping

    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return iter(self._data)

    def __getitem__(self, key):
        value = self._data[key]
        if isinstance(value, basestring):
            # TODO: May need to be different depending on OS
            value = value.replace('"', '\\"')
        return value

    def __setitem__(self, key, value):
        self._data[key] = value

    def __delitem__(self, key):
        del self._data[key]

    def __copy__(self):
        return EscapingDict(self._data.copy())

class PluginExec(object):
    """
    Execute commands

    Simple example, xecute command for entries that reach output::

      exec: echo 'found {{title}} at {{url}}' > file

    Advanced Example::

      exec:
        on_start:
          phase: echo "Started"
        on_input:
          for_entries: echo 'got {{title}}'
        on_output:
          for_accepted: echo 'accepted {{title}} - {{url}} > file

    You can use all (available) entry fields in the command.
    """

    NAME = 'exec'
    HANDLED_PHASES = ['start', 'input', 'filter', 'output', 'exit']

    schema = {
        'oneOf': [
            one_or_more({'type': 'string'}),
            {
                'type': 'object',
                'properties': {
                    'on_start': {'$ref': '#/definitions/phaseSettings'},
                    'on_input': {'$ref': '#/definitions/phaseSettings'},
                    'on_filter': {'$ref': '#/definitions/phaseSettings'},
                    'on_output': {'$ref': '#/definitions/phaseSettings'},
                    'on_exit': {'$ref': '#/definitions/phaseSettings'},
                    'fail_entries': {'type': 'boolean'},
                    'auto_escape': {'type': 'boolean'},
                    'encoding': {'type': 'string'},
                    'allow_background': {'type': 'boolean'}
                },
                'additionalProperties': False
            }
        ],
        'definitions': {
            'phaseSettings': {
                'type': 'object',
                'properties': {
                    'phase': one_or_more({'type': 'string'}),
                    'for_entries': one_or_more({'type': 'string'}),
                    'for_accepted': one_or_more({'type': 'string'}),
                    'for_rejected': one_or_more({'type': 'string'}),
                    'for_failed': one_or_more({'type': 'string'})
                },
                'additionalProperties': False
            }
        }
    }

    def prepare_config(self, config):
        if isinstance(config, basestring):
            config = {'on_output': {'for_accepted': config}}
        if not config.get('encoding'):
            config['encoding'] = io_encoding
        for phase_name in config:
            if phase_name.startswith('on_'):
                for items_name in config[phase_name]:
                    if isinstance(config[phase_name][items_name], basestring):
                        config[phase_name][items_name] = [config[phase_name][items_name]]

        return config

    def execute_cmd(self, cmd, allow_background, encoding):
        log.verbose('Executing: %s' % cmd)
        p = subprocess.Popen(cmd.encode(encoding), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, close_fds=False)
        if not allow_background:
            (r, w) = (p.stdout, p.stdin)
            response = r.read().decode(encoding, 'replace')
            r.close()
            w.close()
            if response:
                log.info('Stdout: %s' % response)
        return p.wait()

    def execute(self, task, phase_name, config):
        config = self.prepare_config(config)
        if not phase_name in config:
            log.debug('phase %s not configured' % phase_name)
            return

        name_map = {'for_entries': task.entries, 'for_accepted': task.accepted,
                    'for_rejected': task.rejected, 'for_failed': task.failed}

        allow_background = config.get('allow_background')
        for operation, entries in name_map.iteritems():
            if not operation in config[phase_name]:
                continue

            log.debug('running phase_name: %s operation: %s entries: %s' % (phase_name, operation, len(entries)))

            for entry in entries:
                for cmd in config[phase_name][operation]:
                    entrydict = EscapingDict(entry) if config.get('auto_escape') else entry
                    # Do string replacement from entry, but make sure quotes get escaped
                    try:
                        cmd = render_from_entry(cmd, entrydict)
                    except RenderError as e:
                        log.error('Could not set exec command for %s: %s' % (entry['title'], e))
                        # fail the entry if configured to do so
                        if config.get('fail_entries'):
                            entry.fail('Entry `%s` does not have required fields for string replacement.' % entry['title'])
                        continue

                    log.debug('phase_name: %s operation: %s cmd: %s' % (phase_name, operation, cmd))
                    if task.options.test:
                        log.info('Would execute: %s' % cmd)
                    else:
                        # Make sure the command can be encoded into appropriate encoding, don't actually encode yet,
                        # so logging continues to work.
                        try:
                            cmd.encode(config['encoding'])
                        except UnicodeEncodeError:
                            log.error('Unable to encode cmd `%s` to %s' % (cmd, config['encoding']))
                            if config.get('fail_entries'):
                                entry.fail('cmd `%s` could not be encoded to %s.' % (cmd, config['encoding']))
                            continue
                        # Run the command, fail entries with non-zero return code if configured to
                        if self.execute_cmd(cmd, allow_background, config['encoding']) != 0 and config.get('fail_entries'):
                            entry.fail('exec return code was non-zero')

        # phase keyword in this
        if 'phase' in config[phase_name]:
            for cmd in config[phase_name]['phase']:
                try:
                    cmd = render_from_task(cmd, task)
                except RenderError as e:
                    log.error('Error rendering `%s`: %s' % (cmd, e))
                else:
                    log.debug('phase cmd: %s' % cmd)
                    if task.options.test:
                        log.info('Would execute: %s' % cmd)
                    else:
                        self.execute_cmd(cmd, allow_background, config['encoding'])

    def __getattr__(self, item):
        """Creates methods to handle task phases."""
        for phase in self.HANDLED_PHASES:
            if item == plugin.phase_methods[phase]:
                # A phase method we handle has been requested
                break
        else:
            # We don't handle this phase
            raise AttributeError(item)

        def phase_handler(task, config):
            self.execute(task, 'on_' + phase, config)

        # Make sure we run after other plugins so exec can use their output
        phase_handler.priority = 100
        return phase_handler

@event('plugin.register')
def register_plugin():
    plugin.register(PluginExec, 'exec', api_ver=2)
<commit_msg>fix exec list<commit_after>from __future__ import unicode_literals, division, absolute_import
from collections import MutableMapping
import logging
import subprocess

from flexget import plugin
from flexget.event import event
from flexget.config_schema import one_or_more
from flexget.utils.template import render_from_entry, render_from_task, RenderError
from flexget.utils.tools import io_encoding

log = logging.getLogger('exec')

class EscapingDict(MutableMapping):
    """Helper class, same as a dict, but returns all string value with quotes escaped."""

    def __init__(self, mapping):
        self._data = mapping

    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return iter(self._data)

    def __getitem__(self, key):
        value = self._data[key]
        if isinstance(value, basestring):
            # TODO: May need to be different depending on OS
            value = value.replace('"', '\\"')
        return value

    def __setitem__(self, key, value):
        self._data[key] = value

    def __delitem__(self, key):
        del self._data[key]

    def __copy__(self):
        return EscapingDict(self._data.copy())

class PluginExec(object):
    """
    Execute commands

    Simple example, xecute command for entries that reach output::

      exec: echo 'found {{title}} at {{url}}' > file

    Advanced Example::

      exec:
        on_start:
          phase: echo "Started"
        on_input:
          for_entries: echo 'got {{title}}'
        on_output:
          for_accepted: echo 'accepted {{title}} - {{url}} > file

    You can use all (available) entry fields in the command.
    """

    NAME = 'exec'
    HANDLED_PHASES = ['start', 'input', 'filter', 'output', 'exit']

    schema = {
        'oneOf': [
            one_or_more({'type': 'string'}),
            {
                'type': 'object',
                'properties': {
                    'on_start': {'$ref': '#/definitions/phaseSettings'},
                    'on_input': {'$ref': '#/definitions/phaseSettings'},
                    'on_filter': {'$ref': '#/definitions/phaseSettings'},
                    'on_output': {'$ref': '#/definitions/phaseSettings'},
                    'on_exit': {'$ref': '#/definitions/phaseSettings'},
                    'fail_entries': {'type': 'boolean'},
                    'auto_escape': {'type': 'boolean'},
                    'encoding': {'type': 'string'},
                    'allow_background': {'type': 'boolean'}
                },
                'additionalProperties': False
            }
        ],
        'definitions': {
            'phaseSettings': {
                'type': 'object',
                'properties': {
                    'phase': one_or_more({'type': 'string'}),
                    'for_entries': one_or_more({'type': 'string'}),
                    'for_accepted': one_or_more({'type': 'string'}),
                    'for_rejected': one_or_more({'type': 'string'}),
                    'for_failed': one_or_more({'type': 'string'})
                },
                'additionalProperties': False
            }
        }
    }

    def prepare_config(self, config):
        if isinstance(config, basestring):
            config = [config]
        if isinstance(config, list):
            config = {'on_output': {'for_accepted': config}}
        if not config.get('encoding'):
            config['encoding'] = io_encoding
        for phase_name in config:
            if phase_name.startswith('on_'):
                for items_name in config[phase_name]:
                    if isinstance(config[phase_name][items_name], basestring):
                        config[phase_name][items_name] = [config[phase_name][items_name]]

        return config

    def execute_cmd(self, cmd, allow_background, encoding):
        log.verbose('Executing: %s' % cmd)
        p = subprocess.Popen(cmd.encode(encoding), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, close_fds=False)
        if not allow_background:
            (r, w) = (p.stdout, p.stdin)
            response = r.read().decode(encoding, 'replace')
            r.close()
            w.close()
            if response:
                log.info('Stdout: %s' % response)
        return p.wait()

    def execute(self, task, phase_name, config):
        config = self.prepare_config(config)
        if not phase_name in config:
            log.debug('phase %s not configured' % phase_name)
            return

        name_map = {'for_entries': task.entries, 'for_accepted': task.accepted,
                    'for_rejected': task.rejected, 'for_failed': task.failed}

        allow_background = config.get('allow_background')
        for operation, entries in name_map.iteritems():
            if not operation in config[phase_name]:
                continue

            log.debug('running phase_name: %s operation: %s entries: %s' % (phase_name, operation, len(entries)))

            for entry in entries:
                for cmd in config[phase_name][operation]:
                    entrydict = EscapingDict(entry) if config.get('auto_escape') else entry
                    # Do string replacement from entry, but make sure quotes get escaped
                    try:
                        cmd = render_from_entry(cmd, entrydict)
                    except RenderError as e:
                        log.error('Could not set exec command for %s: %s' % (entry['title'], e))
                        # fail the entry if configured to do so
                        if config.get('fail_entries'):
                            entry.fail('Entry `%s` does not have required fields for string replacement.' % entry['title'])
                        continue

                    log.debug('phase_name: %s operation: %s cmd: %s' % (phase_name, operation, cmd))
                    if task.options.test:
                        log.info('Would execute: %s' % cmd)
                    else:
                        # Make sure the command can be encoded into appropriate encoding, don't actually encode yet,
                        # so logging continues to work.
                        try:
                            cmd.encode(config['encoding'])
                        except UnicodeEncodeError:
                            log.error('Unable to encode cmd `%s` to %s' % (cmd, config['encoding']))
                            if config.get('fail_entries'):
                                entry.fail('cmd `%s` could not be encoded to %s.' % (cmd, config['encoding']))
                            continue
                        # Run the command, fail entries with non-zero return code if configured to
                        if self.execute_cmd(cmd, allow_background, config['encoding']) != 0 and config.get('fail_entries'):
                            entry.fail('exec return code was non-zero')

        # phase keyword in this
        if 'phase' in config[phase_name]:
            for cmd in config[phase_name]['phase']:
                try:
                    cmd = render_from_task(cmd, task)
                except RenderError as e:
                    log.error('Error rendering `%s`: %s' % (cmd, e))
                else:
                    log.debug('phase cmd: %s' % cmd)
                    if task.options.test:
                        log.info('Would execute: %s' % cmd)
                    else:
                        self.execute_cmd(cmd, allow_background, config['encoding'])

    def __getattr__(self, item):
        """Creates methods to handle task phases."""
        for phase in self.HANDLED_PHASES:
            if item == plugin.phase_methods[phase]:
                # A phase method we handle has been requested
                break
        else:
            # We don't handle this phase
            raise AttributeError(item)

        def phase_handler(task, config):
            self.execute(task, 'on_' + phase, config)

        # Make sure we run after other plugins so exec can use their output
        phase_handler.priority = 100
        return phase_handler

@event('plugin.register')
def register_plugin():
    plugin.register(PluginExec, 'exec', api_ver=2)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Temporary patches
-----------------

"""

from random import randint
from time import sleep

from numpy.linalg.linalg import LinAlgError
from nipype.algorithms import confounds as nac
from nipype.interfaces.base import (File, traits)
from nipype.interfaces.mixins import reporting

class RetryCompCorInputSpecMixin(reporting.ReportCapableInputSpec):
    out_report = File('report.html', usedefault=True, hash_files=False,
                      desc='filename for warning HTML snippet')
    # 'NaN` by default
    failure_mode = traits.Enum(
        'NaN', 'error',
        usedefault=True,
        desc='When no components are found or convergence fails, raise an error '
             'or silently return columns of NaNs.')

class RetryCompCorMixin(reporting.ReportCapableInterface):
    def _run_interface(self, runtime):
        warn = self.inputs.failure_mode == 'NaN'

        failures = 0
        save_exc = None
        while True:
            success = True
            # Identifiy success/failure in both error and NaN mode
            try:
                runtime = super()._run_interface(runtime)
                if warn and self._check_nans():
                    success = False
            except LinAlgError as exc:
                success = False
                save_exc = exc

            if success:
                break

            failures += 1
            if failures > 10:
                if warn:
                    break
                raise save_exc
            start = (failures - 1) * 10
            sleep(randint(start + 4, start + 10))

        return runtime

    def _check_nans(self):
        import numpy as np
        outputs = self._list_outputs()
        components = np.loadtxt(outputs['components_file'], skiprows=1)
        return np.isnan(components).all()

    def _generate_report(self):
        snippet = ''.format(self._header)
        if self._check_nans():
            snippet = '''\

    Warning: {} components could not be estimated, due to a linear algebra error.
    While not definitive, this may be an indication of a poor mask.
    Please inspect the {} contours above to ensure that they are located
    in the white matter/CSF.

'''.format(self._header, 'magenta' if self._header[0] == 'a' else 'blue')

        with open(self._out_report, 'w') as fobj:
            fobj.write(snippet)

class RobustACompCorInputSpec(RetryCompCorInputSpecMixin, nac.CompCorInputSpec):
    pass

class RobustACompCorOutputSpec(reporting.ReportCapableOutputSpec, nac.CompCorOutputSpec):
    pass

class RobustACompCor(RetryCompCorMixin, nac.ACompCor):
    """
    Runs aCompCor several times if it suddenly fails with
    https://github.com/poldracklab/fmriprep/issues/776

    Warns by default, rather than failing, on linear algebra errors.
    https://github.com/poldracklab/fmriprep/issues/1433

    """
    input_spec = RobustACompCorInputSpec
    output_spec = RobustACompCorOutputSpec

class RobustTCompCorInputSpec(RetryCompCorInputSpecMixin, nac.TCompCorInputSpec):
    pass

class RobustTCompCorOutputSpec(reporting.ReportCapableOutputSpec, nac.TCompCorOutputSpec):
    pass

class RobustTCompCor(RetryCompCorMixin, nac.ACompCor):
    """
    Runs tCompCor several times if it suddenly fails with
    https://github.com/poldracklab/fmriprep/issues/776

    Warns by default, rather than failing, on linear algebra errors.
    https://github.com/poldracklab/fmriprep/issues/1433

    """
    input_spec = RobustTCompCorInputSpec
    output_spec = RobustTCompCorOutputSpec
<commit_msg>STY: _check_nans -> _is_allnans<commit_after># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Temporary patches
-----------------

"""

from random import randint
from time import sleep

from numpy.linalg.linalg import LinAlgError
from nipype.algorithms import confounds as nac
from nipype.interfaces.base import (File, traits)
from nipype.interfaces.mixins import reporting

class RetryCompCorInputSpecMixin(reporting.ReportCapableInputSpec):
    out_report = File('report.html', usedefault=True, hash_files=False,
                      desc='filename for warning HTML snippet')
    # 'NaN` by default
    failure_mode = traits.Enum(
        'NaN', 'error',
        usedefault=True,
        desc='When no components are found or convergence fails, raise an error '
             'or silently return columns of NaNs.')

class RetryCompCorMixin(reporting.ReportCapableInterface):
    def _run_interface(self, runtime):
        warn = self.inputs.failure_mode == 'NaN'

        failures = 0
        save_exc = None
        while True:
            success = True
            # Identifiy success/failure in both error and NaN mode
            try:
                runtime = super()._run_interface(runtime)
                if warn and self._is_allnans():
                    success = False
            except LinAlgError as exc:
                success = False
                save_exc = exc

            if success:
                break

            failures += 1
            if failures > 10:
                if warn:
                    break
                raise save_exc
            start = (failures - 1) * 10
            sleep(randint(start + 4, start + 10))

        return runtime

    def _is_allnans(self):
        import numpy as np
        outputs = self._list_outputs()
        components = np.loadtxt(outputs['components_file'], skiprows=1)
        return np.isnan(components).all()

    def _generate_report(self):
        snippet = ''.format(self._header)
        if self._is_allnans():
            snippet = '''\

    Warning: {} components could not be estimated, due to a linear algebra error.
    While not definitive, this may be an indication of a poor mask.
    Please inspect the {} contours above to ensure that they are located
    in the white matter/CSF.

'''.format(self._header, 'magenta' if self._header[0] == 'a' else 'blue')

        with open(self._out_report, 'w') as fobj:
            fobj.write(snippet)

class RobustACompCorInputSpec(RetryCompCorInputSpecMixin, nac.CompCorInputSpec):
    pass

class RobustACompCorOutputSpec(reporting.ReportCapableOutputSpec, nac.CompCorOutputSpec):
    pass

class RobustACompCor(RetryCompCorMixin, nac.ACompCor):
    """
    Runs aCompCor several times if it suddenly fails with
    https://github.com/poldracklab/fmriprep/issues/776

    Warns by default, rather than failing, on linear algebra errors.
    https://github.com/poldracklab/fmriprep/issues/1433

    """
    input_spec = RobustACompCorInputSpec
    output_spec = RobustACompCorOutputSpec

class RobustTCompCorInputSpec(RetryCompCorInputSpecMixin, nac.TCompCorInputSpec):
    pass

class RobustTCompCorOutputSpec(reporting.ReportCapableOutputSpec, nac.TCompCorOutputSpec):
    pass

class RobustTCompCor(RetryCompCorMixin, nac.ACompCor):
    """
    Runs tCompCor several times if it suddenly fails with
    https://github.com/poldracklab/fmriprep/issues/776

    Warns by default, rather than failing, on linear algebra errors.
    https://github.com/poldracklab/fmriprep/issues/1433

    """
    input_spec = RobustTCompCorInputSpec
    output_spec = RobustTCompCorOutputSpec
<|endoftext|>