<commit_before>from django.db import models
from django.utils.translation import ugettext as _, ugettext_lazy as _l
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse

from notorhot.fields import AutoDocumentableImageField

from autoslug import AutoSlugField
from model_utils import Choices
from model_utils.managers import PassThroughManager

import datetime

class PublicCategoryManager(models.Manager):
    """
    Manager for :class:`CandidateCategory` that returns only instances with
    :attr:`~CandidateCategory.is_public` ``== True``
    """
    def get_queryset(self):
        return super(PublicCategoryManager, self).get_queryset().filter(
            is_public=True)

class CandidateCategory(models.Model):
    """
    Category of "things" ("Candidates") to compare.  :class:`Candidate`
    instances belonging to different
    :class:`CandidateCategories <CandidateCategory>` instances should never be
    offered up against one another in the same :class:`Competition`.
    """
    name = models.CharField(max_length=50)
    slug = AutoSlugField(populate_from='name', unique=True, blank=True)
    is_public = models.BooleanField(default=True, help_text=_l(u"If false, "
        u"this category will not be listed, and competitions for this category "
        u"will not be available."))

    objects = models.Manager()
    public = PublicCategoryManager()

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('notorhot_competition', kwargs={ 'slug': self.slug, })

    def get_leaderboard_url(self):
        """
        URL for leaderboard for this category
        """
        return reverse('notorhot_leaders', kwargs={ 'category_slug': self.slug, })

    def generate_competition(self):
        """
        :returns: a :class:`Competition` instance with two :class:`Candidate`
            instances selected at random from this :class:`CandidateCategory`.
        :rtype: :class:`Competition`
        """
        return Competition.objects.generate_from_queryset(
            self.candidates.enabled())

    @property
    def num_candidates(self):
        """
        :returns: number of enabled :class:`Candidate` instances belonging to
            this :class:`CandidateCategory`.
        :rtype: integer
        """
        return self.candidates.enabled().count()

    def num_voted_competitions(self):
        """
        :returns: number of :class`Competition` instances in this
            :class:`CandidateCategory` that have received votes.
        :rtype: integer
        """
        return self.competitions.count() - self.competitions.votable().count()
    num_voted_competitions = property(num_voted_competitions, doc="Competitions voted in")

    class Meta:
        ordering = ('name',)

class CandidateQuerySet(models.query.QuerySet):
    """
    :class:`QuerySet` that adds additional ordering and filtering shortcut
        methods for :class:`Candidate` sets.
    """
    def order_by_wins(self):
        """
        :returns: :class:`Candidate` queryset sorted by calculated win
            percentage (wins/votes).
        :rtype: :class:`QuerySet`
        """
        return self.extra(select={ 'win_pct': 'wins / votes', }).order_by(
            '-win_pct')

    def for_category(self, category):
        """
        :arg category: :class:`CandidateCategory` by which the queryset should be
            filtered
        :returns: queryset filtered to include only :class:`Candidate`
            instances belonging to the specified category.
        :rtype: :class:`QuerySet`
        """
        return self.filter(category=category)

    def enabled(self):
        """
        :returns: :class:`Candidate` queryset filtered to include ony instances
            with :attr:`~Candidate.is_enabled` ``== True``
        :rtype: :class:`QuerySet`
        """
        return self.filter(is_enabled=True)

CandidateManager = PassThroughManager.for_queryset_class(CandidateQuerySet)

class EnabledCandidateManager(CandidateManager):
    """
    Manager that returns only :class:`Candidate` instances having
    :attr:`~Candidate.is_enabled` ``== True``
    """
    def get_queryset(self):
        return super(EnabledCandidateManager, self).get_queryset().enabled()

class Candidate(models.Model):
    """
    Option to be voted on.  For instance, if our site lets us vote for the
    awesomest fruit, we might have a :class:`Candidate` instance for Apples
    and one for Bananas.
    """
    name = models.CharField(max_length=100)
    slug = AutoSlugField(populate_from='name', unique=True, blank=True)
    pic = AutoDocumentableImageField(upload_to='candidates')
    is_enabled = models.BooleanField(default=True)

    category = models.ForeignKey(CandidateCategory, related_name='candidates')

    # These fields technically violate 3rd normal form, but are a lot less
    # expensive to update than to calculate.
    challenges = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of times this candidate has been presented to users"))
    votes = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of times users have submitted votes in comparison "
            u"that included this candidate."))
    wins = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of competitions this candidate has won"))

    added = models.DateTimeField(auto_now_add=True)

    objects = CandidateManager()
    enabled = EnabledCandidateManager()

    def __unicode__(self):
        return self.name

    def increment_challenges(self):
        self.challenges += 1
        self.save()

    def increment_votes(self, won):
        """
        Increments :attr:`~Candidate.votes` field.  If ``won == True``, also
        increments :attr:`~Candidate.wins` field.

        :arg boolean won: Whether or not this was the winning candidate in
            the vote.
        """
        self.votes += 1
        if won:
            self.wins += 1
        self.save()

    @property
    def win_percentage(self):
        """
        Calculated win rate (:attr:`~Candidate.wins`/:attr:`~Candidate.votes`)
        """
        if self.votes == 0:
            return 'No votes'

        return 100 * float(self.wins)/float(self.votes)

    def get_absolute_url(self):
        return reverse('notorhot_candidate', kwargs={
            'slug': self.slug,
            'category_slug': self.category.slug,
        })

class CompetitionQuerySet(models.query.QuerySet):
    """
    QuerySet that adds additional filtering shortcut methods for
    :class:`Competition` instances.
    """
    def votable(self):
        return self.filter(date_voted__isnull=True)

CompetitionManager = PassThroughManager.for_queryset_class(CompetitionQuerySet)

class CompetitionGeneratingManager(CompetitionManager):
    """
    :class:`CompetitionManager` with additional methods to generate
    :class:`Competition` instances with randomized or specified
    :class:`Candidate` instances.
    """
    def generate_from_candidates(self, left, right):
        """
        :arg left: :class:`Candidate` instance for the left side of the vote
            display
        :arg right: :class:`Candidate` instance for the right side of the vote
            display
        :returns: a new (not yet voted on) :class:`Competition` instance with
            the specified "left" and "right" :class:`Candidate` instances.
        :rtype: :class:`Competition`
        """
        return self.create(**{
            'left': left,
            'right': right,
        })

    def generate_from_queryset(self, queryset):
        """
        Selects two :class:`Candidate` instances at random from the specified
        queryset, and creates a new :class:`Competition` between them.

        :arg queryset: pool of :class:`Candidate` instances from which to select
        :type queryset: :class:`QuerySet`
        :returns: new :class:`Competition` between two :class:`Candidate`
            instances selected at random from ``queryset``
        :rtype: :class:`Competition`

        .. warning::
            If passed a queryset containing :class:`Candidate` instances
            belonging to multiple :class:`Categories <CandidateCategory>`,
            this method is liable to generate a :class:`Competition` between
            :class:`Candidates <Candidate>` belonging to different categories,
            which is technically in violation of our business constraints.  No
            validation currently exists to prevent this case.
        """
        # randomize queryset order, then select first two objects
        first_2 = queryset.order_by('?')[:2]

        # Not enough candidates available total
        if len(first_2) < 2:
            raise Candidate.DoesNotExist

        return self.generate_from_candidates(first_2[0], first_2[1])

    def generate(self):
        """
        Selects two :class:`Candidate` instances at random from the pool of all
        :class:`Candidate` instances, and creates a new :class:`Competition`
        between them.

        :returns: new :class:`Competition` between two :class:`Candidate`
            instances selected at random
        :rtype: :class:`Competition`

        .. warning::
            Use at your own risk.

            This method is liable tto generate a :class:`Competition` between
            :class:`Candidates <Candidate>` belonging to different categories,
            which is technically in violation of our business constraints.  No
            validation currently exists to prevent this case.
        """
        return self.generate_from_queryset(Candidate.enabled.all())

class VotableCompetitionManager(CompetitionManager):
    """
    Manager that returns only :class:`Competition` instances that have not yet
    been voted on (having :attr:`~Competition.date_voted` ``is None``)
    """
    def get_queryset(self):
        return super(VotableCompetitionManager, self).get_queryset().votable()

class Competition(models.Model):
    """
    A comparison between two :clas::`Candidate` instances ("left" and "right"
    candidates) and -- if the user submits a vote -- the result of that vote.
    """

    SIDES = Choices(
        (1, 'LEFT', _l(u"Left")),
        (2, 'RIGHT', _l(u"Right")),
    )

    date_presented = models.DateTimeField(auto_now_add=True)
    date_voted = models.DateTimeField(null=True, blank=True)

    left = models.ForeignKey(Candidate, related_name='comparisons_left',
        help_text=_l(u"Candidate to present on the left-hand side of the comparison"))
    right = models.ForeignKey(Candidate, related_name='comparisons_right',
        help_text=_l(u"Candidate to present on the right-hand side of the comparison"))

    # denormalized for easier filtering
    category = models.ForeignKey(CandidateCategory, related_name='competitions')

    winner = models.ForeignKey(Candidate, related_name='comparisons_won',
        null=True, blank=True,
        help_text=_l(u"Candidate who won the competition"))

    # denormalized for easier reporting
    winning_side = models.PositiveSmallIntegerField(null=True, blank=True,
        choices=SIDES)

    objects = CompetitionGeneratingManager()
    votable = VotableCompetitionManager()

    def __unicode__(self):
        return "%s vs. %s" % (self.left, self.right)

    def clean(self):
        """
        Validates that the winner is one of the :class:`Candidate` instances
        belonging to this :class:`Competition` and that the competition's
        category matches both of its candidates' categories.
        """
        if self.winner and self.winner.id not in (self.left.id, self.right.id):
            raise ValidationError(_(u"Winner must be one of the candidates "
                u"offered on left or right."))

        if ((self.left.category != self.right.category) or
                (self.category and (self.left.category != self.category))):
            raise ValidationError(_(u"Both candidates for competition must "
                u"belong to same category as competition."))

    def save(self, *args, **kwargs):
        """
        Upon :class:`Competition` instance creation, increments challenge count
        on both its candidates; if category is blank, sets a value.
        """
        if not self.id:
            self.left.increment_challenges()
            self.right.increment_challenges()

        try:
            self.category
        except CandidateCategory.DoesNotExist:
            self.category = self.left.category

        super(Competition, self).save(*args, **kwargs)

    def record_vote(self, winner):
        """
        Records the winning candidate on the :class:`Competition`.  Also updates
        statistics on both :class:`Candidate` records.
        """
        self.winning_side = winner

        if winner == self.SIDES.LEFT:
            self.winner = self.left
        elif winner == self.SIDES.RIGHT:
            self.winner = self.right

        self.left.increment_votes((winner == self.SIDES.LEFT))
        self.right.increment_votes((winner == self.SIDES.RIGHT))

        self.date_voted = datetime.datetime.now()

        self.save()

<commit_msg>MySQL casts the result of integer division to a Decimal, but SQLite treats it as an integer and rounds down.  Since SQLite calls its decimal/float types "REAL" instead of "DECIMAL", accommodate both by multiplying the numerator by 1.0 to force the entire expression to a float/decimal type.<commit_after>from django.db import models
from django.utils.translation import ugettext as _, ugettext_lazy as _l
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse

from notorhot.fields import AutoDocumentableImageField

from autoslug import AutoSlugField
from model_utils import Choices
from model_utils.managers import PassThroughManager

import datetime

class PublicCategoryManager(models.Manager):
    """
    Manager for :class:`CandidateCategory` that returns only instances with
    :attr:`~CandidateCategory.is_public` ``== True``
    """
    def get_queryset(self):
        return super(PublicCategoryManager, self).get_queryset().filter(
            is_public=True)

class CandidateCategory(models.Model):
    """
    Category of "things" ("Candidates") to compare.  :class:`Candidate`
    instances belonging to different
    :class:`CandidateCategories <CandidateCategory>` instances should never be
    offered up against one another in the same :class:`Competition`.
    """
    name = models.CharField(max_length=50)
    slug = AutoSlugField(populate_from='name', unique=True, blank=True)
    is_public = models.BooleanField(default=True, help_text=_l(u"If false, "
        u"this category will not be listed, and competitions for this category "
        u"will not be available."))

    objects = models.Manager()
    public = PublicCategoryManager()

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('notorhot_competition', kwargs={ 'slug': self.slug, })

    def get_leaderboard_url(self):
        """
        URL for leaderboard for this category
        """
        return reverse('notorhot_leaders', kwargs={ 'category_slug': self.slug, })

    def generate_competition(self):
        """
        :returns: a :class:`Competition` instance with two :class:`Candidate`
            instances selected at random from this :class:`CandidateCategory`.
        :rtype: :class:`Competition`
        """
        return Competition.objects.generate_from_queryset(
            self.candidates.enabled())

    @property
    def num_candidates(self):
        """
        :returns: number of enabled :class:`Candidate` instances belonging to
            this :class:`CandidateCategory`.
        :rtype: integer
        """
        return self.candidates.enabled().count()

    def num_voted_competitions(self):
        """
        :returns: number of :class`Competition` instances in this
            :class:`CandidateCategory` that have received votes.
        :rtype: integer
        """
        return self.competitions.count() - self.competitions.votable().count()
    num_voted_competitions = property(num_voted_competitions, doc="Competitions voted in")

    class Meta:
        ordering = ('name',)

class CandidateQuerySet(models.query.QuerySet):
    """
    :class:`QuerySet` that adds additional ordering and filtering shortcut
        methods for :class:`Candidate` sets.
    """
    def order_by_wins(self):
        """
        :returns: :class:`Candidate` queryset sorted by calculated win
            percentage (wins/votes).
        :rtype: :class:`QuerySet`
        """
        return self.extra(select={ 'win_pct': '1.0 * wins / votes', }).order_by(
            '-win_pct')

    def for_category(self, category):
        """
        :arg category: :class:`CandidateCategory` by which the queryset should be
            filtered
        :returns: queryset filtered to include only :class:`Candidate`
            instances belonging to the specified category.
        :rtype: :class:`QuerySet`
        """
        return self.filter(category=category)

    def enabled(self):
        """
        :returns: :class:`Candidate` queryset filtered to include ony instances
            with :attr:`~Candidate.is_enabled` ``== True``
        :rtype: :class:`QuerySet`
        """
        return self.filter(is_enabled=True)

CandidateManager = PassThroughManager.for_queryset_class(CandidateQuerySet)

class EnabledCandidateManager(CandidateManager):
    """
    Manager that returns only :class:`Candidate` instances having
    :attr:`~Candidate.is_enabled` ``== True``
    """
    def get_queryset(self):
        return super(EnabledCandidateManager, self).get_queryset().enabled()

class Candidate(models.Model):
    """
    Option to be voted on.  For instance, if our site lets us vote for the
    awesomest fruit, we might have a :class:`Candidate` instance for Apples
    and one for Bananas.
    """
    name = models.CharField(max_length=100)
    slug = AutoSlugField(populate_from='name', unique=True, blank=True)
    pic = AutoDocumentableImageField(upload_to='candidates')
    is_enabled = models.BooleanField(default=True)

    category = models.ForeignKey(CandidateCategory, related_name='candidates')

    # These fields technically violate 3rd normal form, but are a lot less
    # expensive to update than to calculate.
    challenges = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of times this candidate has been presented to users"))
    votes = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of times users have submitted votes in comparison "
            u"that included this candidate."))
    wins = models.PositiveIntegerField(default=0, blank=True,
        help_text=_l(u"Number of competitions this candidate has won"))

    added = models.DateTimeField(auto_now_add=True)

    objects = CandidateManager()
    enabled = EnabledCandidateManager()

    def __unicode__(self):
        return self.name

    def increment_challenges(self):
        self.challenges += 1
        self.save()

    def increment_votes(self, won):
        """
        Increments :attr:`~Candidate.votes` field.  If ``won == True``, also
        increments :attr:`~Candidate.wins` field.

        :arg boolean won: Whether or not this was the winning candidate in
            the vote.
        """
        self.votes += 1
        if won:
            self.wins += 1
        self.save()

    @property
    def win_percentage(self):
        """
        Calculated win rate (:attr:`~Candidate.wins`/:attr:`~Candidate.votes`)
        """
        if self.votes == 0:
            return 'No votes'

        return 100 * float(self.wins)/float(self.votes)

    def get_absolute_url(self):
        return reverse('notorhot_candidate', kwargs={
            'slug': self.slug,
            'category_slug': self.category.slug,
        })

class CompetitionQuerySet(models.query.QuerySet):
    """
    QuerySet that adds additional filtering shortcut methods for
    :class:`Competition` instances.
    """
    def votable(self):
        return self.filter(date_voted__isnull=True)

CompetitionManager = PassThroughManager.for_queryset_class(CompetitionQuerySet)

class CompetitionGeneratingManager(CompetitionManager):
    """
    :class:`CompetitionManager` with additional methods to generate
    :class:`Competition` instances with randomized or specified
    :class:`Candidate` instances.
    """
    def generate_from_candidates(self, left, right):
        """
        :arg left: :class:`Candidate` instance for the left side of the vote
            display
        :arg right: :class:`Candidate` instance for the right side of the vote
            display
        :returns: a new (not yet voted on) :class:`Competition` instance with
            the specified "left" and "right" :class:`Candidate` instances.
        :rtype: :class:`Competition`
        """
        return self.create(**{
            'left': left,
            'right': right,
        })

    def generate_from_queryset(self, queryset):
        """
        Selects two :class:`Candidate` instances at random from the specified
        queryset, and creates a new :class:`Competition` between them.

        :arg queryset: pool of :class:`Candidate` instances from which to select
        :type queryset: :class:`QuerySet`
        :returns: new :class:`Competition` between two :class:`Candidate`
            instances selected at random from ``queryset``
        :rtype: :class:`Competition`

        .. warning::
            If passed a queryset containing :class:`Candidate` instances
            belonging to multiple :class:`Categories <CandidateCategory>`,
            this method is liable to generate a :class:`Competition` between
            :class:`Candidates <Candidate>` belonging to different categories,
            which is technically in violation of our business constraints.  No
            validation currently exists to prevent this case.
        """
        # randomize queryset order, then select first two objects
        first_2 = queryset.order_by('?')[:2]

        # Not enough candidates available total
        if len(first_2) < 2:
            raise Candidate.DoesNotExist

        return self.generate_from_candidates(first_2[0], first_2[1])

    def generate(self):
        """
        Selects two :class:`Candidate` instances at random from the pool of all
        :class:`Candidate` instances, and creates a new :class:`Competition`
        between them.

        :returns: new :class:`Competition` between two :class:`Candidate`
            instances selected at random
        :rtype: :class:`Competition`

        .. warning::
            Use at your own risk.

            This method is liable tto generate a :class:`Competition` between
            :class:`Candidates <Candidate>` belonging to different categories,
            which is technically in violation of our business constraints.  No
            validation currently exists to prevent this case.
        """
        return self.generate_from_queryset(Candidate.enabled.all())

class VotableCompetitionManager(CompetitionManager):
    """
    Manager that returns only :class:`Competition` instances that have not yet
    been voted on (having :attr:`~Competition.date_voted` ``is None``)
    """
    def get_queryset(self):
        return super(VotableCompetitionManager, self).get_queryset().votable()

class Competition(models.Model):
    """
    A comparison between two :clas::`Candidate` instances ("left" and "right"
    candidates) and -- if the user submits a vote -- the result of that vote.
    """

    SIDES = Choices(
        (1, 'LEFT', _l(u"Left")),
        (2, 'RIGHT', _l(u"Right")),
    )

    date_presented = models.DateTimeField(auto_now_add=True)
    date_voted = models.DateTimeField(null=True, blank=True)

    left = models.ForeignKey(Candidate, related_name='comparisons_left',
        help_text=_l(u"Candidate to present on the left-hand side of the comparison"))
    right = models.ForeignKey(Candidate, related_name='comparisons_right',
        help_text=_l(u"Candidate to present on the right-hand side of the comparison"))

    # denormalized for easier filtering
    category = models.ForeignKey(CandidateCategory, related_name='competitions')

    winner = models.ForeignKey(Candidate, related_name='comparisons_won',
        null=True, blank=True,
        help_text=_l(u"Candidate who won the competition"))

    # denormalized for easier reporting
    winning_side = models.PositiveSmallIntegerField(null=True, blank=True,
        choices=SIDES)

    objects = CompetitionGeneratingManager()
    votable = VotableCompetitionManager()

    def __unicode__(self):
        return "%s vs. %s" % (self.left, self.right)

    def clean(self):
        """
        Validates that the winner is one of the :class:`Candidate` instances
        belonging to this :class:`Competition` and that the competition's
        category matches both of its candidates' categories.
        """
        if self.winner and self.winner.id not in (self.left.id, self.right.id):
            raise ValidationError(_(u"Winner must be one of the candidates "
                u"offered on left or right."))

        if ((self.left.category != self.right.category) or
                (self.category and (self.left.category != self.category))):
            raise ValidationError(_(u"Both candidates for competition must "
                u"belong to same category as competition."))

    def save(self, *args, **kwargs):
        """
        Upon :class:`Competition` instance creation, increments challenge count
        on both its candidates; if category is blank, sets a value.
        """
        if not self.id:
            self.left.increment_challenges()
            self.right.increment_challenges()

        try:
            self.category
        except CandidateCategory.DoesNotExist:
            self.category = self.left.category

        super(Competition, self).save(*args, **kwargs)

    def record_vote(self, winner):
        """
        Records the winning candidate on the :class:`Competition`.  Also updates
        statistics on both :class:`Candidate` records.
        """
        self.winning_side = winner

        if winner == self.SIDES.LEFT:
            self.winner = self.left
        elif winner == self.SIDES.RIGHT:
            self.winner = self.right

        self.left.increment_votes((winner == self.SIDES.LEFT))
        self.right.increment_votes((winner == self.SIDES.RIGHT))

        self.date_voted = datetime.datetime.now()

        self.save()

<|endoftext|>


<commit_before>### Copyright (2012) Continuum Analytics, Inc
### All Rights Reserved

import sys, os, logging
import subprocess, tempfile
import functools

import numba.pycc as pyc

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

def get_ending(args):
    if args.llvm:
        return ".bc"
    if args.olibs:
        return ".o"
    else:
        return pyc.find_shared_ending()

def main(args=[]):
    if not args:
        args = sys.argv
    import argparse
    parser = argparse.ArgumentParser(description="Compile Python modules to a single shared library")
    parser.add_argument("inputs", nargs='+', help="Input file(s)")
    parser.add_argument("-o", nargs=1, dest="output", help="Output file")
    parser.add_argument("-c", action="store_true", dest="olibs",
                        help="Create object file from each input instead of shared-library")
    parser.add_argument("--llvm", action="store_true",
                        help="Emit llvm instead of native code")
    parser.add_argument("--linker", nargs=1, help="Path to linker (default is platform dependent)")
    parser.add_argument("--linker-args", help="Arguments to pass to linker")
    parser.add_argument('--header', action="store_true",
                        help="Emit C header file with function signatures")
    parser.add_argument('--python', action='store_true',
                        help='Emit additionally generated Python wrapper and '
                        'extension module code in output')

    if os.path.basename(args[0]) in ['pycc.py', 'pycc']:
        args = args[1:]

    args = parser.parse_args(args)
    if args.output:
        args.output = args.output[0]
        output_base = os.path.split(args.output)[1]
        module_name = os.path.splitext(output_base)[0]
    else:
        input_base = os.path.splitext(args.inputs[0])[0]
        module_name = os.path.split(input_base)[1]
        args.output = input_base + get_ending(args)
    logger.debug('args.output --> %s', args.output)

    # run the compiler
    logger.debug('inputs --> %s', args.inputs)
    compiler = pyc.Compiler(args.inputs, module_name=module_name)
    if args.llvm:
        logger.debug('emit llvm')
        compiler.write_llvm_bitcode(args.output, wrap=args.python)
    elif args.olibs:
        logger.debug('emit object file')
        compiler.write_native_object(args.output, wrap=args.python)
    else:
        logger.debug('emit shared library')
        logger.debug('write to temporary object file %s', tempfile.gettempdir())
        temp_obj = (tempfile.gettempdir() + os.sep +
                    os.path.basename(args.output) + '.o')
        compiler.write_native_object(temp_obj, wrap=args.python)
        cmdargs = (pyc.find_linker(),) + pyc.find_args() + ('-o', args.output, temp_obj)
        subprocess.check_call(cmdargs)
        os.remove(temp_obj)   # remove temporary object

    if args.header:
        compiler.emit_header(args.output)

<commit_msg>Keep pycc from setting the logging level (make it quiet).<commit_after>### Copyright (2012) Continuum Analytics, Inc
### All Rights Reserved

import sys, os, logging
import subprocess, tempfile
import functools

import numba.pycc as pyc

#logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

def get_ending(args):
    if args.llvm:
        return ".bc"
    if args.olibs:
        return ".o"
    else:
        return pyc.find_shared_ending()

def main(args=[]):
    if not args:
        args = sys.argv
    import argparse
    parser = argparse.ArgumentParser(description="Compile Python modules to a single shared library")
    parser.add_argument("inputs", nargs='+', help="Input file(s)")
    parser.add_argument("-o", nargs=1, dest="output", help="Output file")
    parser.add_argument("-c", action="store_true", dest="olibs",
                        help="Create object file from each input instead of shared-library")
    parser.add_argument("--llvm", action="store_true",
                        help="Emit llvm instead of native code")
    parser.add_argument("--linker", nargs=1, help="Path to linker (default is platform dependent)")
    parser.add_argument("--linker-args", help="Arguments to pass to linker")
    parser.add_argument('--header', action="store_true",
                        help="Emit C header file with function signatures")
    parser.add_argument('--python', action='store_true',
                        help='Emit additionally generated Python wrapper and '
                        'extension module code in output')

    if os.path.basename(args[0]) in ['pycc.py', 'pycc']:
        args = args[1:]

    args = parser.parse_args(args)
    if args.output:
        args.output = args.output[0]
        output_base = os.path.split(args.output)[1]
        module_name = os.path.splitext(output_base)[0]
    else:
        input_base = os.path.splitext(args.inputs[0])[0]
        module_name = os.path.split(input_base)[1]
        args.output = input_base + get_ending(args)
    logger.debug('args.output --> %s', args.output)

    # run the compiler
    logger.debug('inputs --> %s', args.inputs)
    compiler = pyc.Compiler(args.inputs, module_name=module_name)
    if args.llvm:
        logger.debug('emit llvm')
        compiler.write_llvm_bitcode(args.output, wrap=args.python)
    elif args.olibs:
        logger.debug('emit object file')
        compiler.write_native_object(args.output, wrap=args.python)
    else:
        logger.debug('emit shared library')
        logger.debug('write to temporary object file %s', tempfile.gettempdir())
        temp_obj = (tempfile.gettempdir() + os.sep +
                    os.path.basename(args.output) + '.o')
        compiler.write_native_object(temp_obj, wrap=args.python)
        cmdargs = (pyc.find_linker(),) + pyc.find_args() + ('-o', args.output, temp_obj)
        subprocess.check_call(cmdargs)
        os.remove(temp_obj)   # remove temporary object

    if args.header:
        compiler.emit_header(args.output)

<|endoftext|>


<commit_before>import os, sys
sys.path.append(os.getcwd())

import time

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import sklearn.datasets
import tensorflow as tf

import tflib as lib
import tflib.save_images
import tflib.mnist
import tflib.plot

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
from torch.autograd import Variable

import models.dcgan as dcgan
import models.mlp as mlp

DIM = 64 # Model dimensionality
BATCH_SIZE = 50 # Batch size
CRITIC_ITERS = 5 # For WGAN and WGAN-GP, number of critic iters per gen iter
LAMBDA = 10 # Gradient penalty lambda hyperparameter
ITERS = 200000 # How many generator iterations to train for
OUTPUT_DIM = 784 # Number of pixels in MNIST (28*28)

lib.print_model_settings(locals().copy())

# ==================Definition Start======================
class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()

        preprocess = nn.Sequential(
            nn.Linear(128, 4*4*4*DIM),
            nn.ReLU(True),
        )
        block1 = nn.Sequential(
            nn.ConvTranspose2d(4*DIM, 2*DIM, 5),
            nn.ReLU(True),
        )
        block2 = nn.Sequential(
            nn.ConvTranspose2d(2*DIM, DIM, 5),
            nn.ReLU(True),
        )
        deconv_out = nn.ConvTranspose2d(DIM, 1, 5)

        self.block1 = block1
        self.block2 = block2
        self.deconv_out = deconv_out
        self.preprocess = preprocess
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        output = self.preprocess(input)
        output = output.view(-1, 4*DIM, 4, 4)
        output = self.block1(output)
        output = output[:, :, :7, :7]
        output = self.block2(output)
        output = self.deconv_out(output)
        output = self.sigmoid(output)
        return output.view(-1, OUTPUT_DIM)

# TODO: Replace this using CNN once pytorch is supported
class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        main = nn.Sequential(
            # Z goes into a linear of size: ndf
            nn.Linear(OUTPUT_DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 1),
        )
        self.main = main

    def forward(self, input):
        return self.main(input).view(-1)

def generate_image(frame, netG):
    noise = torch.randn(BATCH_SIZE, 128)
    if use_cuda:
        noise = noise.cuda()
    noisev = autograd.Variable(noise, volatile=True)
    samples = netG(noisev)
    samples = samples.view(128, 28, 28)
    # print samples.size()

    samples = samples.cpu().data.numpy()

    lib.save_images.save_images(
        samples,
        'samples_{}.png'.format(frame)
    )

# Dataset iterator
train_gen, dev_gen, test_gen = lib.mnist.load(BATCH_SIZE, BATCH_SIZE)
def inf_train_gen():
    while True:
        for images,targets in train_gen():
            yield images

def calc_gradient_penalty(netD, real_data, fake_data):
    alpha = torch.rand(BATCH_SIZE, 1, 1)
    alpha = alpha.expand(real_data.size())
    alpha = alpha.cuda() if use_cuda else alpha

    interpolates = alpha * real_data + ((1 - alpha) * fake_data)

    if use_cuda:
        interpolates = interpolates.cuda()
    interpolates = autograd.Variable(interpolates, requires_grad=True)

    disc_interpolates = netD(interpolates)

    # TODO: Make ConvBackward diffentiable
    gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
                              grad_outputs=torch.ones(disc_interpolates.size()).cuda() if use_cuda else torch.ones(
                                  disc_interpolates.size()),
                              create_graph=True, retain_graph=True, only_inputs=True)[0]

    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA
    return gradient_penalty

# ==================Definition End======================

netG = Generator()
netD = Discriminator()
print netG
print netD

if use_cuda:
    netD = netD.cuda()
    netG = netG.cuda()

optimizerD = optim.Adam(netD.parameters(), lr=1e-4, betas=(0.5, 0.9))
optimizerG = optim.Adam(netG.parameters(), lr=1e-4, betas=(0.5, 0.9))

one = torch.FloatTensor([1])
mone = one * -1
if use_cuda:
    one = one.cuda()
    mone = mone.cuda()

data = inf_train_gen()

for iteration in xrange(ITERS):
    start_time = time.time()
    ############################
    # (1) Update D network
    ###########################
    for p in netD.parameters():  # reset requires_grad
        p.requires_grad = True  # they are set to False below in netG update

    for iter_d in xrange(CRITIC_ITERS):
        _data = data.next()
        real_data = torch.Tensor(_data)
        if use_cuda:
            real_data = real_data.cuda()
        real_data_v = autograd.Variable(real_data)

        netD.zero_grad()

        # train with real
        D_real = netD(real_data_v)
        D_real = D_real.mean()
        # print D_real
        D_real.backward(mone)

        # train with fake
        noise = torch.randn(BATCH_SIZE, 128)
        if use_cuda:
            noise = noise.cuda()
        noisev = autograd.Variable(noise, volatile=True)  # totally freeze netG
        fake = autograd.Variable(netG(noisev).data)
        inputv = fake
        D_fake = netD(inputv)
        D_fake = D_fake.mean()
        D_fake.backward(one)

        # train with gradient penalty
        gradient_penalty = calc_gradient_penalty(netD, real_data_v.data, fake.data)
        gradient_penalty.backward()

        D = D_fake - D_real + gradient_penalty
        D_cost = -D
        optimizerD.step()

    ############################
    # (2) Update G network
    ###########################
    for p in netD.parameters():
        p.requires_grad = False  # to avoid computation
    netG.zero_grad()

    noise = torch.randn(BATCH_SIZE, 128)
    if use_cuda:
        noise = noise.cuda()
    noisev = autograd.Variable(noise)
    fake = netG(noisev)
    G = netD(fake)
    G = G.mean()
    G.backward(mone)
    G_cost = -G
    optimizerG.step()

    # Write logs and save samples
    lib.plot.plot('time', time.time() - start_time)
    lib.plot.plot('train disc cost', D_cost.cpu().data.numpy())
    lib.plot.plot('train gen cost', G_cost.cpu().data.numpy())

    # Calculate dev loss and generate samples every 100 iters
    if iteration % 100 == 99:
        dev_disc_costs = []
        for images,_ in dev_gen():
            imgs = torch.Tensor(images)
            if use_cuda:
                imgs = imgs.cuda()
            imgs_v = autograd.Variable(imgs, volatile=True)

            D = netD(real_data_v)
            _dev_disc_cost = -D.mean().cpu().data.numpy()
            dev_disc_costs.append(_dev_disc_cost)
        lib.plot.plot('dev disc cost', np.mean(dev_disc_costs))

        generate_image(iteration, netG)

    # Write logs every 100 iters
    if (iteration < 5) or (iteration % 100 == 99):
        lib.plot.flush()

    lib.plot.tick()
<commit_msg>Finish gan_mnist.py<commit_after>import os, sys
sys.path.append(os.getcwd())

import time

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import sklearn.datasets
import tensorflow as tf

import tflib as lib
import tflib.save_images
import tflib.mnist
import tflib.plot

import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

torch.manual_seed(1)
use_cuda = torch.cuda.is_available()

DIM = 64 # Model dimensionality
BATCH_SIZE = 50 # Batch size
CRITIC_ITERS = 5 # For WGAN and WGAN-GP, number of critic iters per gen iter
LAMBDA = 10 # Gradient penalty lambda hyperparameter
ITERS = 200000 # How many generator iterations to train for
OUTPUT_DIM = 784 # Number of pixels in MNIST (28*28)

lib.print_model_settings(locals().copy())

# ==================Definition Start======================
class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()

        preprocess = nn.Sequential(
            nn.Linear(128, 4*4*4*DIM),
            nn.ReLU(True),
        )
        block1 = nn.Sequential(
            nn.ConvTranspose2d(4*DIM, 2*DIM, 5),
            nn.ReLU(True),
        )
        block2 = nn.Sequential(
            nn.ConvTranspose2d(2*DIM, DIM, 5),
            nn.ReLU(True),
        )
        deconv_out = nn.ConvTranspose2d(DIM, 1, 8, stride=2)

        self.block1 = block1
        self.block2 = block2
        self.deconv_out = deconv_out
        self.preprocess = preprocess
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        output = self.preprocess(input)
        output = output.view(-1, 4*DIM, 4, 4)
        #print output.size()
        output = self.block1(output)
        #print output.size()
        output = output[:, :, :7, :7]
        #print output.size()
        output = self.block2(output)
        #print output.size()
        output = self.deconv_out(output)
        output = self.sigmoid(output)
        #print output.size()
        return output.view(-1, OUTPUT_DIM)

# TODO: Replace this using CNN once pytorch is supported
class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        main = nn.Sequential(
            nn.Linear(OUTPUT_DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 4*4*4*DIM),
            nn.ReLU(True),
            nn.Linear(4*4*4*DIM, 1),
        )
        self.main = main

    def forward(self, input):
        return self.main(input).view(-1)

def generate_image(frame, netG):
    noise = torch.randn(BATCH_SIZE, 128)
    if use_cuda:
        noise = noise.cuda()
    noisev = autograd.Variable(noise, volatile=True)
    samples = netG(noisev)
    samples = samples.view(BATCH_SIZE, 28, 28)
    # print samples.size()

    samples = samples.cpu().data.numpy()

    lib.save_images.save_images(
        samples,
        'tmp/mnist/samples_{}.png'.format(frame)
    )

# Dataset iterator
train_gen, dev_gen, test_gen = lib.mnist.load(BATCH_SIZE, BATCH_SIZE)
def inf_train_gen():
    while True:
        for images,targets in train_gen():
            yield images

def calc_gradient_penalty(netD, real_data, fake_data):
    #print real_data.size()
    alpha = torch.rand(BATCH_SIZE, 1)
    alpha = alpha.expand(real_data.size())
    alpha = alpha.cuda() if use_cuda else alpha

    interpolates = alpha * real_data + ((1 - alpha) * fake_data)

    if use_cuda:
        interpolates = interpolates.cuda()
    interpolates = autograd.Variable(interpolates, requires_grad=True)

    disc_interpolates = netD(interpolates)

    # TODO: Make ConvBackward diffentiable
    gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
                              grad_outputs=torch.ones(disc_interpolates.size()).cuda() if use_cuda else torch.ones(
                                  disc_interpolates.size()),
                              create_graph=True, retain_graph=True, only_inputs=True)[0]

    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA
    return gradient_penalty

# ==================Definition End======================

netG = Generator()
netD = Discriminator()
print netG
print netD

if use_cuda:
    netD = netD.cuda()
    netG = netG.cuda()

optimizerD = optim.Adam(netD.parameters(), lr=1e-4, betas=(0.5, 0.9))
optimizerG = optim.Adam(netG.parameters(), lr=1e-4, betas=(0.5, 0.9))

one = torch.FloatTensor([1])
mone = one * -1
if use_cuda:
    one = one.cuda()
    mone = mone.cuda()

data = inf_train_gen()

for iteration in xrange(ITERS):
    start_time = time.time()
    ############################
    # (1) Update D network
    ###########################
    for p in netD.parameters():  # reset requires_grad
        p.requires_grad = True  # they are set to False below in netG update

    for iter_d in xrange(CRITIC_ITERS):
        _data = data.next()
        real_data = torch.Tensor(_data)
        if use_cuda:
            real_data = real_data.cuda()
        real_data_v = autograd.Variable(real_data)

        netD.zero_grad()

        # train with real
        D_real = netD(real_data_v)
        D_real = D_real.mean()
        # print D_real
        D_real.backward(mone)

        # train with fake
        noise = torch.randn(BATCH_SIZE, 128)
        if use_cuda:
            noise = noise.cuda()
        noisev = autograd.Variable(noise, volatile=True)  # totally freeze netG
        fake = autograd.Variable(netG(noisev).data)
        inputv = fake
        D_fake = netD(inputv)
        D_fake = D_fake.mean()
        D_fake.backward(one)

        # train with gradient penalty
        gradient_penalty = calc_gradient_penalty(netD, real_data_v.data, fake.data)
        gradient_penalty.backward()

        D_cost = D_fake - D_real + gradient_penalty
        optimizerD.step()

    ############################
    # (2) Update G network
    ###########################
    for p in netD.parameters():
        p.requires_grad = False  # to avoid computation
    netG.zero_grad()

    noise = torch.randn(BATCH_SIZE, 128)
    if use_cuda:
        noise = noise.cuda()
    noisev = autograd.Variable(noise)
    fake = netG(noisev)
    G = netD(fake)
    G = G.mean()
    G.backward(mone)
    G_cost = -G
    optimizerG.step()

    # Write logs and save samples
    lib.plot.plot('tmp/mnist/time', time.time() - start_time)
    lib.plot.plot('tmp/mnist/train disc cost', D_cost.cpu().data.numpy())
    lib.plot.plot('tmp/mnist/train gen cost', G_cost.cpu().data.numpy())

    # Calculate dev loss and generate samples every 100 iters
    if iteration % 100 == 99:
        dev_disc_costs = []
        for images,_ in dev_gen():
            imgs = torch.Tensor(images)
            if use_cuda:
                imgs = imgs.cuda()
            imgs_v = autograd.Variable(imgs, volatile=True)

            D = netD(imgs_v)
            _dev_disc_cost = -D.mean().cpu().data.numpy()
            dev_disc_costs.append(_dev_disc_cost)
        lib.plot.plot('tmp/mnist/dev disc cost', np.mean(dev_disc_costs))

        generate_image(iteration, netG)

    # Write logs every 100 iters
    if (iteration < 5) or (iteration % 100 == 99):
        lib.plot.flush()

    lib.plot.tick()
<|endoftext|>


<commit_before>"""
ActionCable connection.
"""

import threading
import uuid
import json
import logging
import time
import websocket

class Connection:
    """
    The connection to a websocket server
    """
    def __init__(self, url, origin=None, log_ping=False, cookie=None, header=None):
        """
        :param url: The url of the cable server.
        :param origin: (Optional) The origin.
        :param log_ping: (Default: False) If true every
                                            ping gets logged.
        :param cookie: (Optional) A cookie to send (used for
                                            authentication for instance).
        :param header: (Optional) custom header for websocket handshake.
        """
        self.url = url
        self.origin = origin
        self.log_ping = log_ping
        self.cookie = cookie
        self.header = header if header is not None else []

        self.logger = logging.getLogger('ActionCable Connection')

        self.subscriptions = {}

        self.websocket = None
        self.ws_thread = None

        self.auto_reconnect = False

        if origin is not None:
            self.origin = origin

    def connect(self, origin=None):
        """
        Connects to the server.

        :param origin: (Optional) The origin.
        """
        self.logger.debug('Establish connection...')

        if self.connected:
            self.logger.warning('Connection already established. Return...')
            return

        if origin is not None:
            self.origin = origin

        self.auto_reconnect = True

        self.ws_thread = threading.Thread(
            name="APIConnectionThread_{}".format(uuid.uuid1()),
            target=self._run_forever)
        self.ws_thread.daemon = True
        self.ws_thread.start()

    def disconnect(self):
        """
        Closes the connection.
        """
        self.logger.debug('Close connection...')

        self.auto_reconnect = False

        if self.websocket is not None:
            self.websocket.close()

    def _run_forever(self):
        while self.auto_reconnect:
            try:
                self.logger.debug('Run connection loop.')

                self.websocket = websocket.WebSocketApp(
                    self.url,
                    cookie=self.cookie,
                    header=self.header,
                    on_message=self._on_message,
                    on_close=self._on_close)
                self.websocket.on_open = self._on_open

                self.websocket.run_forever(ping_interval=5, ping_timeout=3, origin=self.origin)
                time.sleep(2)
            except Exception as exc:
                self.logger.error('Connection loop raised exception. Exception: %s', exc)

    def send(self, data):
        """
        Sends data to the server.
        """
        self.logger.debug('Send data: {}'.format(data))

        if not self.connected:
            self.logger.warning('Connection not established. Return...')
            return

        self.websocket.send(json.dumps(data))

    def _on_open(self, socket):
        """
        Called when the connection is open.
        """
        self.logger.debug('Connection established.')

    def _on_message(self, socket, message):
        """
        Called aways when a message arrives.
        """
        data = json.loads(message)
        message_type = None
        identifier = None
        subscription = None

        if 'type' in data:
            message_type = data['type']

        if 'identifier' in data:
            identifier = json.loads(data['identifier'])

        if identifier is not None:
            subscription = self.find_subscription(identifier)

        if subscription is not None:
            subscription.received(data)
        elif message_type == 'welcome':
            self.logger.debug('Welcome message received.')

            for subscription in self.subscriptions.values():
                if subscription.state == 'connection_pending':
                    subscription.create()

        elif message_type == 'ping':
            if self.log_ping:
                self.logger.debug('Ping received.')
        else:
            self.logger.warning('Message not supported. (Message: {})'.format(message))

    def _on_close(self, socket):
        """
        Called when the connection was closed.
        """
        self.logger.debug('Connection closed.')

        for subscription in self.subscriptions.values():
            if subscription.state == 'subscribed':
                subscription.state = 'connection_pending'

    @property
    def socket_present(self):
        """
        If socket is present.
        """
        return self.websocket is not None and self.websocket.sock is not None

    @property
    def connected(self):
        """
        If connected to server.
        """
        return self.websocket is not None and \
               self.websocket.sock is not None and \
               self.websocket.sock.connected

    def find_subscription(self, identifier):
        """
        Finds a subscription
        by it's identifier.
        """
        for subscription in self.subscriptions.values():
            if subscription.identifier == identifier:
                return subscription
<commit_msg>Update connection.py<commit_after>"""
ActionCable connection.
"""

import threading
import uuid
import json
import logging
import time
import websocket

class Connection:
    """
    The connection to a websocket server
    """
    def __init__(self, url, origin=None, log_ping=False, cookie=None, header=None):
        """
        :param url: The url of the cable server.
        :param origin: (Optional) The origin.
        :param log_ping: (Default: False) If true every
                                            ping gets logged.
        :param cookie: (Optional) A cookie to send (used for
                                            authentication for instance).
        :param header: (Optional) custom header for websocket handshake.
        """
        self.url = url
        self.origin = origin
        self.log_ping = log_ping
        self.cookie = cookie
        self.header = header

        self.logger = logging.getLogger('ActionCable Connection')

        self.subscriptions = {}

        self.websocket = None
        self.ws_thread = None

        self.auto_reconnect = False

        if origin is not None:
            self.origin = origin

    def connect(self, origin=None):
        """
        Connects to the server.

        :param origin: (Optional) The origin.
        """
        self.logger.debug('Establish connection...')

        if self.connected:
            self.logger.warning('Connection already established. Return...')
            return

        if origin is not None:
            self.origin = origin

        self.auto_reconnect = True

        self.ws_thread = threading.Thread(
            name="APIConnectionThread_{}".format(uuid.uuid1()),
            target=self._run_forever)
        self.ws_thread.daemon = True
        self.ws_thread.start()

    def disconnect(self):
        """
        Closes the connection.
        """
        self.logger.debug('Close connection...')

        self.auto_reconnect = False

        if self.websocket is not None:
            self.websocket.close()

    def _run_forever(self):
        while self.auto_reconnect:
            try:
                self.logger.debug('Run connection loop.')

                self.websocket = websocket.WebSocketApp(
                    self.url,
                    cookie=self.cookie,
                    header=self.header,
                    on_message=self._on_message,
                    on_close=self._on_close)
                self.websocket.on_open = self._on_open

                self.websocket.run_forever(ping_interval=5, ping_timeout=3, origin=self.origin)
                time.sleep(2)
            except Exception as exc:
                self.logger.error('Connection loop raised exception. Exception: %s', exc)

    def send(self, data):
        """
        Sends data to the server.
        """
        self.logger.debug('Send data: {}'.format(data))

        if not self.connected:
            self.logger.warning('Connection not established. Return...')
            return

        self.websocket.send(json.dumps(data))

    def _on_open(self, socket):
        """
        Called when the connection is open.
        """
        self.logger.debug('Connection established.')

    def _on_message(self, socket, message):
        """
        Called aways when a message arrives.
        """
        data = json.loads(message)
        message_type = None
        identifier = None
        subscription = None

        if 'type' in data:
            message_type = data['type']

        if 'identifier' in data:
            identifier = json.loads(data['identifier'])

        if identifier is not None:
            subscription = self.find_subscription(identifier)

        if subscription is not None:
            subscription.received(data)
        elif message_type == 'welcome':
            self.logger.debug('Welcome message received.')

            for subscription in self.subscriptions.values():
                if subscription.state == 'connection_pending':
                    subscription.create()

        elif message_type == 'ping':
            if self.log_ping:
                self.logger.debug('Ping received.')
        else:
            self.logger.warning('Message not supported. (Message: {})'.format(message))

    def _on_close(self, socket):
        """
        Called when the connection was closed.
        """
        self.logger.debug('Connection closed.')

        for subscription in self.subscriptions.values():
            if subscription.state == 'subscribed':
                subscription.state = 'connection_pending'

    @property
    def socket_present(self):
        """
        If socket is present.
        """
        return self.websocket is not None and self.websocket.sock is not None

    @property
    def connected(self):
        """
        If connected to server.
        """
        return self.websocket is not None and \
               self.websocket.sock is not None and \
               self.websocket.sock.connected

    def find_subscription(self, identifier):
        """
        Finds a subscription
        by it's identifier.
        """
        for subscription in self.subscriptions.values():
            if subscription.identifier == identifier:
                return subscription
<|endoftext|>


<commit_before>import logging
import sys
from contextlib import contextmanager
from datetime import datetime
from os import getcwd
from os.path import join

LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(message)s'
DATE_FORMAT = '[%Y-%m-%d %H:%M:%S]'

logger = logging.getLogger()
logger.level = logging.DEBUG

def configure_stream_logger(stream=sys.stdout, level='DEBUG'):
    stream_handler = logging.StreamHandler(stream=stream)
    stream_handler.level = logging.getLevelName(level)
    formatter = logging.Formatter(datefmt=DATE_FORMAT, fmt=LOGGING_FORMAT)
    stream_handler.setFormatter(formatter)
    logging.getLogger().addHandler(stream_handler)

def configure_file_logger(filename='app.log', level='DEBUG', rotate_file=True):
    file_path = join(getcwd(), filename)
    file_handler = logging.FileHandler(filename=file_path)
    file_handler.level = logging.getLevelName(level)
    formatter = logging.Formatter(datefmt=DATE_FORMAT, fmt=LOGGING_FORMAT)
    file_handler.setFormatter(formatter)
    logging.getLogger().addHandler(file_handler)
    if rotate_file:
        open(file_path, 'w').close()

def configure_file_and_stream_logger(stream=sys.stdout, filename='app.log', level='DEBUG', rotate_file=True):
    configure_stream_logger(stream, level)
    configure_file_logger(filename, level, rotate_file)

@contextmanager
def timeit():
    start_time = datetime.now()
    logger.info('START TIME: %s', start_time)
    yield
    logger.info('TIME SPENT: %s', (datetime.now() - start_time))
<commit_msg>!U support configured loggers tracking to avoid duplications<commit_after>import logging
import sys
from contextlib import contextmanager
from datetime import datetime
from os import getcwd
from os.path import join

LOGGING_FORMAT = '%(asctime)-15s %(levelname)s %(message)s'
DATE_FORMAT = '[%Y-%m-%d %H:%M:%S]'

logger = logging.getLogger()
logger.level = logging.DEBUG

def configure_stream_logger(stream=sys.stdout, level='DEBUG', track_configured_loggers=True):
    level = logging.getLevelName(level)
    if track_configured_loggers:
        for handler in logging.root.handlers:
            if isinstance(handler, logging.StreamHandler) and handler.stream == stream \
                    and handler.level == level:
                return

    stream_handler = logging.StreamHandler(stream=stream)
    stream_handler.level = level
    formatter = logging.Formatter(datefmt=DATE_FORMAT, fmt=LOGGING_FORMAT)
    stream_handler.setFormatter(formatter)
    logging.getLogger().addHandler(stream_handler)

def configure_file_logger(filename='app.log', level='DEBUG', rotate_file=True):
    file_path = join(getcwd(), filename)
    file_handler = logging.FileHandler(filename=file_path)
    file_handler.level = logging.getLevelName(level)
    formatter = logging.Formatter(datefmt=DATE_FORMAT, fmt=LOGGING_FORMAT)
    file_handler.setFormatter(formatter)
    logging.getLogger().addHandler(file_handler)
    if rotate_file:
        open(file_path, 'w').close()

def configure_file_and_stream_logger(stream=sys.stdout, filename='app.log', level='DEBUG', rotate_file=True):
    configure_stream_logger(stream, level)
    configure_file_logger(filename, level, rotate_file)

@contextmanager
def timeit():
    start_time = datetime.now()
    logger.info('START TIME: %s', start_time)
    yield
    logger.info('TIME SPENT: %s', (datetime.now() - start_time))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

"""
oauthlib.common
~~~~~~~~~~~~~~

This module provides data structures and utilities common
to all implementations of OAuth.
"""

import random
import re
import sys
import time
try:
    from urllib import quote as _quote
    from urllib import unquote as _unquote
    from urllib import urlencode as _urlencode
except ImportError:
    from urllib.parse import quote as _quote
    from urllib.parse import unquote as _unquote
    from urllib.parse import urlencode as _urlencode
try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

UNICODE_ASCII_CHARACTER_SET = ('abcdefghijklmnopqrstuvwxyz'
                               'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                               '0123456789')

always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
               'abcdefghijklmnopqrstuvwxyz'
               '0123456789' '_.-')

PY3 = sys.version_info[0] == 3

if PY3:
    unicode_type = str
    bytes_type = bytes
else:
    unicode_type = unicode
    bytes_type = str

# 'safe' must be bytes (Python 2.6 requires bytes, other versions allow either)
def quote(s, safe=b'/'):
    s = _quote(s.encode('utf-8'), safe)
    # PY3 always returns unicode.  PY2 may return either, depending on whether
    # it had to modify the string.
    if isinstance(s, bytes_type):
        s = s.decode('utf-8')
    return s

def unquote(s):
    s = _unquote(s)
    # PY3 always returns unicode.  PY2 seems to always return what you give it,
    # which differs from quote's behavior.  Just to be safe, make sure it is
    # unicode before we return.
    if isinstance(s, bytes_type):
        s = s.decode('utf-8')
    return s

def urlencode(params):
    utf8_params = encode_params_utf8(params)
    urlencoded = _urlencode(utf8_params)
    if isinstance(urlencoded, unicode_type):  # PY3 returns unicode
        return urlencoded
    else:
        return urlencoded.decode("utf-8")

def encode_params_utf8(params):
    """Ensures that all parameters in a list of 2-element tuples are encoded to
    bytestrings using UTF-8
    """
    encoded = []
    for k, v in params:
        encoded.append((
            k.encode('utf-8') if isinstance(k, unicode_type) else k,
            v.encode('utf-8') if isinstance(v, unicode_type) else v))
    return encoded

def decode_params_utf8(params):
    """Ensures that all parameters in a list of 2-element tuples are decoded to
    unicode using UTF-8.
    """
    decoded = []
    for k, v in params:
        decoded.append((
            k.decode('utf-8') if isinstance(k, bytes_type) else k,
            v.decode('utf-8') if isinstance(v, bytes_type) else v))
    return decoded

urlencoded = set(always_safe) | set('=&;%+~')

def urldecode(query):
    """Decode a query string in x-www-form-urlencoded format into a sequence
    of two-element tuples.

    Unlike urlparse.parse_qsl(..., strict_parsing=True) urldecode will enforce
    correct formatting of the query string by validation. If validation fails
    a ValueError will be raised. urllib.parse_qsl will only raise errors if
    any of name-value pairs omits the equals sign.
    """
    # Check if query contains invalid characters
    if query and not set(query) <= urlencoded:
        raise ValueError('Not a valid urlencoded string.')

    # Check for correctly hex encoded values using a regular expression
    # All encoded values begin with % followed by two hex characters
    # correct = %00, %A0, %0A, %FF
    # invalid = %G0, %5H, %PO
    invalid_hex = '%[^0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]'
    if len(re.findall(invalid_hex, query)):
        raise ValueError('Invalid hex encoding in query string.')

    query = query.decode('utf-8') if isinstance(query, bytes_type) else query
    # We want to allow queries such as "c2" whereas urlparse.parse_qsl
    # with the strict_parsing flag will not.
    params = urlparse.parse_qsl(query, keep_blank_values=True)

    # unicode all the things
    return decode_params_utf8(params)

def extract_params(raw):
    """Extract parameters and return them as a list of 2-tuples.

    Will successfully extract parameters from urlencoded query strings,
    dicts, or lists of 2-tuples. Empty strings/dicts/lists will return an
    empty list of parameters. Any other input will result in a return
    value of None.
    """
    if isinstance(raw, bytes_type) or isinstance(raw, unicode_type):
        try:
            params = urldecode(raw)
        except ValueError:
            params = None
    elif hasattr(raw, '__iter__'):
        try:
            dict(raw)
        except ValueError:
            params = None
        except TypeError:
            params = None
        else:
            params = list(raw.items() if isinstance(raw, dict) else raw)
            params = decode_params_utf8(params)
    else:
        params = None

    return params

def generate_nonce():
    """Generate pseudorandom nonce that is unlikely to repeat.

    Per `section 3.3`_ of the OAuth 1 RFC 5849 spec.
    Per `section 3.2.1`_ of the MAC Access Authentication spec.

    A random 64-bit number is appended to the epoch timestamp for both
    randomness and to decrease the likelihood of collisions.

    .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
    .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
    """
    return unicode_type(unicode_type(random.getrandbits(64)) + generate_timestamp())

def generate_timestamp():
    """Get seconds since epoch (UTC).

    Per `section 3.3`_ of the OAuth 1 RFC 5849 spec.
    Per `section 3.2.1`_ of the MAC Access Authentication spec.

    .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
    .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
    """
    return unicode_type(int(time.time()))

def generate_token(length=30, chars=UNICODE_ASCII_CHARACTER_SET):
    """Generates a non-guessable OAuth token

    OAuth (1 and 2) does not specify the format of tokens except that they
    should be strings of random characters. Tokens should not be guessable
    and entropy when generating the random characters is important. Which is
    why SystemRandom is used instead of the default random.choice method.
    """
    rand = random.SystemRandom()
    return ''.join(rand.choice(chars) for x in range(length))

def add_params_to_qs(query, params):
    """Extend a query with a list of two-tuples."""
    if isinstance(query, dict):
        queryparams = query.items()
    else:
        queryparams = urlparse.parse_qsl(query, keep_blank_values=True)
    queryparams.extend(params)
    return urlencode(queryparams)

def add_params_to_uri(uri, params, fragment=False):
    """Add a list of two-tuples to the uri query components."""
    sch, net, path, par, query, fra = urlparse.urlparse(uri)
    if fragment:
        fra = add_params_to_qs(query, params)
    else:
        query = add_params_to_qs(query, params)
    return urlparse.urlunparse((sch, net, path, par, query, fra))

def safe_string_equals(a, b):
    """ Near-constant time string comparison.

    Used in order to avoid timing attacks on sensitive information such
    as secret keys during request verification (`rootLabs`_).

    .. _`rootLabs`: http://rdist.root.org/2010/01/07/timing-independent-array-comparison/

    """
    if len(a) != len(b):
        return False

    result = 0
    for x, y in zip(a, b):
        result |= ord(x) ^ ord(y)
    return result == 0

def to_unicode(data, encoding):
    """Convert a number of different types of objects to unicode."""
    if isinstance(data, unicode_type):
        return data

    if isinstance(data, bytes_type):
        return unicode_type(data, encoding=encoding)

    if hasattr(data, '__iter__'):
        try:
            dict(data)
        except TypeError:
            pass
        except ValueError:
            # Assume it's a one dimensional data structure
            return (to_unicode(i, encoding) for i in data)
        else:
            # We support 2.6 which lacks dict comprehensions
            if isinstance(data, dict):
                data = data.items()
            return dict(((to_unicode(k, encoding), to_unicode(v, encoding)) for k, v in data))

    return data

class CaseInsensitiveDict(dict):
    """Basic case insensitive dict with strings only keys."""

    proxy = {}

    def __init__(self, data):
        self.proxy = dict((k.lower(), k) for k in data)
        for k in data:
            self[k] = data[k]

    def __contains__(self, k):
        return k.lower() in self.proxy

    def __delitem__(self, k):
        key = self.proxy[k.lower()]
        super(CaseInsensitiveDict, self).__delitem__(key)
        del self.proxy[k.lower()]

    def __getitem__(self, k):
        key = self.proxy[k.lower()]
        return super(CaseInsensitiveDict, self).__getitem__(key)

    def get(self, k, default=None):
        return self[k] if k in self else default

    def __setitem__(self, k, v):
        super(CaseInsensitiveDict, self).__setitem__(k, v)
        self.proxy[k.lower()] = k

class Request(object):
    """A malleable representation of a signable HTTP request.

    Body argument may contain any data, but parameters will only be decoded if
    they are one of:

    * urlencoded query string
    * dict
    * list of 2-tuples

    Anything else will be treated as raw body data to be passed through
    unmolested.
    """

    def __init__(self, uri, http_method='GET', body=None, headers=None,
            encoding='utf-8'):
        # Convert to unicode using encoding if given, else assume unicode
        encode = lambda x: to_unicode(x, encoding) if encoding else x

        self.uri = encode(uri)
        self.http_method = encode(http_method)
        self.headers = CaseInsensitiveDict(encode(headers or {}))
        self.body = encode(body)
        self.decoded_body = extract_params(encode(body))
        self.oauth_params = []

        self._params = {}
        self._params.update(dict(urldecode(self.uri_query)))
        self._params.update(dict(self.decoded_body or []))
        self._params.update(self.headers)

    def __getattr__(self, name):
        return self._params.get(name, None)

    @property
    def uri_query(self):
        return urlparse.urlparse(self.uri).query

    @property
    def uri_query_params(self):
        return urlparse.parse_qsl(self.uri_query, keep_blank_values=True,
                                  strict_parsing=True)
<commit_msg>Fix decode,encode mixup.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

"""
oauthlib.common
~~~~~~~~~~~~~~

This module provides data structures and utilities common
to all implementations of OAuth.
"""

import random
import re
import sys
import time
try:
    from urllib import quote as _quote
    from urllib import unquote as _unquote
    from urllib import urlencode as _urlencode
except ImportError:
    from urllib.parse import quote as _quote
    from urllib.parse import unquote as _unquote
    from urllib.parse import urlencode as _urlencode
try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

UNICODE_ASCII_CHARACTER_SET = ('abcdefghijklmnopqrstuvwxyz'
                               'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                               '0123456789')

always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
               'abcdefghijklmnopqrstuvwxyz'
               '0123456789' '_.-')

PY3 = sys.version_info[0] == 3

if PY3:
    unicode_type = str
    bytes_type = bytes
else:
    unicode_type = unicode
    bytes_type = str

# 'safe' must be bytes (Python 2.6 requires bytes, other versions allow either)
def quote(s, safe=b'/'):
    s = s.encode('utf-8') if isinstance(s, unicode_type) else s
    s = _quote(s, safe)
    # PY3 always returns unicode.  PY2 may return either, depending on whether
    # it had to modify the string.
    if isinstance(s, bytes_type):
        s = s.decode('utf-8')
    return s

def unquote(s):
    s = _unquote(s)
    # PY3 always returns unicode.  PY2 seems to always return what you give it,
    # which differs from quote's behavior.  Just to be safe, make sure it is
    # unicode before we return.
    if isinstance(s, bytes_type):
        s = s.decode('utf-8')
    return s

def urlencode(params):
    utf8_params = encode_params_utf8(params)
    urlencoded = _urlencode(utf8_params)
    if isinstance(urlencoded, unicode_type):  # PY3 returns unicode
        return urlencoded
    else:
        return urlencoded.decode("utf-8")

def encode_params_utf8(params):
    """Ensures that all parameters in a list of 2-element tuples are encoded to
    bytestrings using UTF-8
    """
    encoded = []
    for k, v in params:
        encoded.append((
            k.encode('utf-8') if isinstance(k, unicode_type) else k,
            v.encode('utf-8') if isinstance(v, unicode_type) else v))
    return encoded

def decode_params_utf8(params):
    """Ensures that all parameters in a list of 2-element tuples are decoded to
    unicode using UTF-8.
    """
    decoded = []
    for k, v in params:
        decoded.append((
            k.decode('utf-8') if isinstance(k, bytes_type) else k,
            v.decode('utf-8') if isinstance(v, bytes_type) else v))
    return decoded

urlencoded = set(always_safe) | set('=&;%+~')

def urldecode(query):
    """Decode a query string in x-www-form-urlencoded format into a sequence
    of two-element tuples.

    Unlike urlparse.parse_qsl(..., strict_parsing=True) urldecode will enforce
    correct formatting of the query string by validation. If validation fails
    a ValueError will be raised. urllib.parse_qsl will only raise errors if
    any of name-value pairs omits the equals sign.
    """
    # Check if query contains invalid characters
    if query and not set(query) <= urlencoded:
        raise ValueError('Not a valid urlencoded string.')

    # Check for correctly hex encoded values using a regular expression
    # All encoded values begin with % followed by two hex characters
    # correct = %00, %A0, %0A, %FF
    # invalid = %G0, %5H, %PO
    invalid_hex = '%[^0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]'
    if len(re.findall(invalid_hex, query)):
        raise ValueError('Invalid hex encoding in query string.')

    query = query.encode('utf-8') if isinstance(query, unicode_type) else query
    # We want to allow queries such as "c2" whereas urlparse.parse_qsl
    # with the strict_parsing flag will not.
    params = urlparse.parse_qsl(query, keep_blank_values=True)

    # unicode all the things
    return decode_params_utf8(params)

def extract_params(raw):
    """Extract parameters and return them as a list of 2-tuples.

    Will successfully extract parameters from urlencoded query strings,
    dicts, or lists of 2-tuples. Empty strings/dicts/lists will return an
    empty list of parameters. Any other input will result in a return
    value of None.
    """
    if isinstance(raw, bytes_type) or isinstance(raw, unicode_type):
        try:
            params = urldecode(raw)
        except ValueError:
            params = None
    elif hasattr(raw, '__iter__'):
        try:
            dict(raw)
        except ValueError:
            params = None
        except TypeError:
            params = None
        else:
            params = list(raw.items() if isinstance(raw, dict) else raw)
            params = decode_params_utf8(params)
    else:
        params = None

    return params

def generate_nonce():
    """Generate pseudorandom nonce that is unlikely to repeat.

    Per `section 3.3`_ of the OAuth 1 RFC 5849 spec.
    Per `section 3.2.1`_ of the MAC Access Authentication spec.

    A random 64-bit number is appended to the epoch timestamp for both
    randomness and to decrease the likelihood of collisions.

    .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
    .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
    """
    return unicode_type(unicode_type(random.getrandbits(64)) + generate_timestamp())

def generate_timestamp():
    """Get seconds since epoch (UTC).

    Per `section 3.3`_ of the OAuth 1 RFC 5849 spec.
    Per `section 3.2.1`_ of the MAC Access Authentication spec.

    .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
    .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
    """
    return unicode_type(int(time.time()))

def generate_token(length=30, chars=UNICODE_ASCII_CHARACTER_SET):
    """Generates a non-guessable OAuth token

    OAuth (1 and 2) does not specify the format of tokens except that they
    should be strings of random characters. Tokens should not be guessable
    and entropy when generating the random characters is important. Which is
    why SystemRandom is used instead of the default random.choice method.
    """
    rand = random.SystemRandom()
    return ''.join(rand.choice(chars) for x in range(length))

def add_params_to_qs(query, params):
    """Extend a query with a list of two-tuples."""
    if isinstance(query, dict):
        queryparams = query.items()
    else:
        queryparams = urlparse.parse_qsl(query, keep_blank_values=True)
    queryparams.extend(params)
    return urlencode(queryparams)

def add_params_to_uri(uri, params, fragment=False):
    """Add a list of two-tuples to the uri query components."""
    sch, net, path, par, query, fra = urlparse.urlparse(uri)
    if fragment:
        fra = add_params_to_qs(query, params)
    else:
        query = add_params_to_qs(query, params)
    return urlparse.urlunparse((sch, net, path, par, query, fra))

def safe_string_equals(a, b):
    """ Near-constant time string comparison.

    Used in order to avoid timing attacks on sensitive information such
    as secret keys during request verification (`rootLabs`_).

    .. _`rootLabs`: http://rdist.root.org/2010/01/07/timing-independent-array-comparison/

    """
    if len(a) != len(b):
        return False

    result = 0
    for x, y in zip(a, b):
        result |= ord(x) ^ ord(y)
    return result == 0

def to_unicode(data, encoding):
    """Convert a number of different types of objects to unicode."""
    if isinstance(data, unicode_type):
        return data

    if isinstance(data, bytes_type):
        return unicode_type(data, encoding=encoding)

    if hasattr(data, '__iter__'):
        try:
            dict(data)
        except TypeError:
            pass
        except ValueError:
            # Assume it's a one dimensional data structure
            return (to_unicode(i, encoding) for i in data)
        else:
            # We support 2.6 which lacks dict comprehensions
            if isinstance(data, dict):
                data = data.items()
            return dict(((to_unicode(k, encoding), to_unicode(v, encoding)) for k, v in data))

    return data

class CaseInsensitiveDict(dict):
    """Basic case insensitive dict with strings only keys."""

    proxy = {}

    def __init__(self, data):
        self.proxy = dict((k.lower(), k) for k in data)
        for k in data:
            self[k] = data[k]

    def __contains__(self, k):
        return k.lower() in self.proxy

    def __delitem__(self, k):
        key = self.proxy[k.lower()]
        super(CaseInsensitiveDict, self).__delitem__(key)
        del self.proxy[k.lower()]

    def __getitem__(self, k):
        key = self.proxy[k.lower()]
        return super(CaseInsensitiveDict, self).__getitem__(key)

    def get(self, k, default=None):
        return self[k] if k in self else default

    def __setitem__(self, k, v):
        super(CaseInsensitiveDict, self).__setitem__(k, v)
        self.proxy[k.lower()] = k

class Request(object):
    """A malleable representation of a signable HTTP request.

    Body argument may contain any data, but parameters will only be decoded if
    they are one of:

    * urlencoded query string
    * dict
    * list of 2-tuples

    Anything else will be treated as raw body data to be passed through
    unmolested.
    """

    def __init__(self, uri, http_method='GET', body=None, headers=None,
            encoding='utf-8'):
        # Convert to unicode using encoding if given, else assume unicode
        encode = lambda x: to_unicode(x, encoding) if encoding else x

        self.uri = encode(uri)
        self.http_method = encode(http_method)
        self.headers = CaseInsensitiveDict(encode(headers or {}))
        self.body = encode(body)
        self.decoded_body = extract_params(encode(body))
        self.oauth_params = []

        self._params = {}
        self._params.update(dict(urldecode(self.uri_query)))
        self._params.update(dict(self.decoded_body or []))
        self._params.update(self.headers)

    def __getattr__(self, name):
        return self._params.get(name, None)

    @property
    def uri_query(self):
        return urlparse.urlparse(self.uri).query

    @property
    def uri_query_params(self):
        return urlparse.parse_qsl(self.uri_query, keep_blank_values=True,
                                  strict_parsing=True)
<|endoftext|>


<commit_before>from clarifai.client import ClarifaiApi
import pdb
from collections import Counter
from collections import defaultdict
import subprocess

def createHashmap(results):
    relevantData = []
    for x in xrange(0,len(results)):
        resultsData = {}
        for y in range(0, len(results[x]['result']['tag']['probs'])):
            resultsData[str(results[x]['result']['tag']['classes'][y])] = results[x]['result']['tag']['probs'][y]
        relevantData.append(resultsData)
    print relevantData
    return relevantData

def main():
    clarifai_api = ClarifaiApi()

    #ADD ANY MORE IMAGES HERE, PROGRAM SHOULD ADJUST
    image_array = [open('output/rgb_img_' + str(x) + ".jpg", 'rb') for x in xrange(1,13)]
    results_json = clarifai_api.tag_images(image_array)
    results = results_json['results']
    hashmap = createHashmap(results)

    find_objects(results)
    text_results_string = str(namesOfObjects)
    espeak(text_results_string)

def espeak(string):
    print string
    subprocess.call(['espeak', "-s", "100", "-v", "+f2", string])

def find_objects(results):
    print results
    for x in xrange(0 , len(results)-1):
        highestInfo = find_highest(x, results)
        prominentObjects.append(highestInfo)
    # checkForPopular(results)

def checkForPopular(results):
    D = defaultdict(list)
    for i,item in enumerate(names):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}
    a = 0

    for x in xrange(0, len(D)):
        indices = D.itervalues().next()
        if len(indices)>2:
            for index in indices:
                results[index]['result']['tag']['concept_ids'].pop(0)
                results[index]['result']['tag']['classes'].pop(0)
                results[index]['result']['tag']['probs'].pop(0)

                highestInfo, timeInHours = find_highest(index, results)

                prominentObjects.pop(index-a)
                namesOfObjects.pop(index-a)
                prominentObjects.insert(index-a, highestInfo)
                namesOfObjects.insert(index-a, str(highestInfo[1]) + " at " + str(int(timeInHours)) + " o'clock.")
                a += 1

def find_highest(x, results):
    hourRatio = ((2.0*x + 1.0)/2.0)/len(results)
    timeInHours = put_time_in_hours(hourRatio)
    highestInfo = []
    effectiveids = results[x]['result']['tag']['concept_ids'] + results[x+1]['result']['tag']['concept_ids']
    effectiveprobs = results[x]['result']['tag']['probs'] + results[x+1]['result']['tag']['probs']
    effectivenames = results[x]['result']['tag']['classes'] + results[x+1]['result']['tag']['classes']
    adjustedprobs = adjust_probs(effectiveprobs)

    duplicate_pairs = find_duplicates(effectiveids)
    adjustedprobs = add_duplicate_probs(duplicate_pairs, adjustedprobs)

    indexOfHighest = find_most_likely(adjustedprobs)
    highestInfo.append(effectiveids[indexOfHighest])
    highestInfo.append(effectivenames[indexOfHighest])
    names.append(str(effectivenames[indexOfHighest]))
    namesOfObjects.append(str(effectivenames[indexOfHighest]) + " at " + str(int(timeInHours)) + " o'clock.")
    highestInfo.append(effectiveprobs[indexOfHighest])

    return highestInfo, timeInHours

def put_time_in_hours(hourRatio):
    hours = round(hourRatio*6, 0)
    adjustedhours = 0
    if hours>3 :
        adjustedhours = hours - 3
    else :
        adjustedhours = hours + 9
    return adjustedhours

def find_most_likely(adjustedprobs):
    index = 0
    for x in xrange(0, len(adjustedprobs)):
        if adjustedprobs[index] < adjustedprobs[x]:
            index = x
    return index

def adjust_probs(effectiveprobs):
    for x in xrange(0,len(effectiveprobs)):
        effectiveprobs[x] -= .85
    return effectiveprobs

def find_duplicates(effectiveids):
    D = defaultdict(list)
    for i,item in enumerate(effectiveids):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}
    return D

def add_duplicate_probs(duplicate_pairs, adjustedprobs):
    for x in xrange(0,len(duplicate_pairs)):
        duplicate_pair = duplicate_pairs.itervalues().next()
        index0 = duplicate_pair[0]
        index1 = duplicate_pair[1]
        adjustedprobs[index0] += adjustedprobs[index1]
        adjustedprobs[index1] = 0
    return adjustedprobs

prominentObjects = []
namesOfObjects = []
names = []

main()

<commit_msg>relevantdata delete not saving<commit_after>from clarifai.client import ClarifaiApi
import pdb
from collections import Counter
from collections import defaultdict
import subprocess

def createHashmap(results):
    relevantData = []
    for x in xrange(0,len(results)):
        resultsData = {}
        for y in range(0, len(results[x]['result']['tag']['probs'])):
            resultsData[str(results[x]['result']['tag']['classes'][y])] = results[x]['result']['tag']['probs'][y]
        relevantData.append(resultsData)
    print relevantData
    return relevantData

def main():
    clarifai_api = ClarifaiApi()

    #ADD ANY MORE IMAGES HERE, PROGRAM SHOULD ADJUST
    image_array = [open('output/rgb_img_' + str(x) + ".jpg", 'rb') for x in xrange(1,13)]
    results_json = clarifai_api.tag_images(image_array)
    results = results_json['results']
    relevantData = createHashmap(results)

    find_objects(relevantData)
    text_results_string = str(namesOfObjects)
    espeak(text_results_string)

def espeak(string):
    print string
    subprocess.call(['espeak', "-s", "100", "-v", "+f2", string])

def find_objects(relevantData):
    del prominentObjects[:]
    del namesOfObjects[:]
    del names[:]
    for x in xrange(0 , len(relevantData)-1):
        highestInfo = find_highest(x, relevantData)
        prominentObjects.append(highestInfo)

    relevantData, isDone = checkForPopular(relevantData)

    while(isDone == False):
        find_objects(relevantData)

def checkForPopular(relevantData):
    D = defaultdict(list)
    for i,item in enumerate(names):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}

    if len(D)!=0 :
        print D
        for x in xrange(0, len(D)):
            indices = D.itervalues().next()
            for index in indices:
                del relevantData[index][relevantData[index].keys()[0]]
        return relevantData, False
    else:
        return relevantData, True

def find_highest(x, relevantData):
    hourRatio = ((2.0*x + 1.0)/2.0)/len(relevantData)
    timeInHours = put_time_in_hours(hourRatio)
    highestInfo = []
    effectiveprobs = relevantData[x].values() + relevantData[x+1].values()
    effectivenames = relevantData[x].keys() + relevantData[x+1].keys()
    adjustedprobs = adjust_probs(effectiveprobs)

    duplicate_pairs = find_duplicates(effectivenames)
    adjustedprobs = add_duplicate_probs(duplicate_pairs, adjustedprobs)

    indexOfHighest = find_most_likely(adjustedprobs)

    highestInfo.append(effectivenames[indexOfHighest])
    highestInfo.append(effectiveprobs[indexOfHighest])

    names.append(str(effectivenames[indexOfHighest]))
    namesOfObjects.append(str(effectivenames[indexOfHighest]) + " at " + str(int(timeInHours)) + " o'clock.")

    return highestInfo

def put_time_in_hours(hourRatio):
    hours = round(hourRatio*6, 0)
    adjustedhours = 0
    if hours>3 :
        adjustedhours = hours - 3
    else :
        adjustedhours = hours + 9
    return adjustedhours

def find_most_likely(adjustedprobs):
    index = 0
    for x in xrange(0, len(adjustedprobs)):
        if adjustedprobs[index] < adjustedprobs[x]:
            index = x
    return index

def adjust_probs(effectiveprobs):
    for x in xrange(0,len(effectiveprobs)):
        effectiveprobs[x] -= .85
    return effectiveprobs

def find_duplicates(effectiveids):
    D = defaultdict(list)
    for i,item in enumerate(effectiveids):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}
    return D

def add_duplicate_probs(duplicate_pairs, adjustedprobs):
    for x in xrange(0,len(duplicate_pairs)):
        duplicate_pair = duplicate_pairs.itervalues().next()
        index0 = duplicate_pair[0]
        index1 = duplicate_pair[1]
        adjustedprobs[index0] += adjustedprobs[index1]
        adjustedprobs[index1] = 0
    return adjustedprobs

prominentObjects = []
namesOfObjects = []
names = []

main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from parsimonious.grammar import Grammar, NodeVisitor

grammar = Grammar(
    """
    clause     = atom ( ":-" _ atom_list)?
    atom_list  = atom  ("," _ atom_list)? _
    atom       = neg? predicate "(" _ term_list ")" _
    neg        = "!" _
    term_list  = term  ("," _ term_list)? _
    term       = constant / variable
    predicate  = low_id / string
    constant   = low_id / string
    variable   = ~"[A-Z][a-z A-Z 0-9_]*"
    xid        = ~"[a-zA-Z_./][a-z A-Z 0-9_./]*"
    low_id     = ~"[a-z_./][a-z A-Z 0-9_./]*"
    string     = ~r"'[^']*'" / ~r"\\"[^\\"]*\\""
    _          = skip*
    skip       = ~r"\s+"
    """)

class Expr:
    def __repr__(self):
        return self.__dict__.__repr__()

    def __eq__(self, other):
        return isinstance(other, self.__class__) and str(other) == str(self)

    def __hash__(self):
        return self.__str__().__hash__()

class Variable(Expr):
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Constant(Expr):
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Predicate(Expr):
    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        return isinstance(other, self.__class__) and other.name == self.name

    def __hash__(self):
        return self.name.__hash__()

    def __repr__(self):
        return self.name

class Atom(Expr):
    def __init__(self, predicate: Predicate, *arguments, negated=False):
        self.predicate = predicate
        self.arguments = arguments
        self.negated = negated

    def __repr__(self):
        return "{}{}({})".format("!" if self.negated else "", self.predicate.name, ", ".join(str(x) for x in self.arguments))

class Clause(Expr):
    def __init__(self, head: Atom, *body):
        self.head = head
        self.body = body

    def __repr__(self):
        return str(self.head) if len(self.body) == 0 else "{head} :- {body}".format(
            head=self.head,
            body=", ".join(str(x) for x in self.body))

class ClauseVisitor(NodeVisitor):
    def visit_clause(self, node, visited_children):
        if len(visited_children[1]) == 0:
            return Clause(visited_children[0])
        else:
            head, ((_, _, body),) = visited_children
            return Clause(head, *body)

    def visit_predicate(self, _, visited_children):
        return Predicate(visited_children[0])

    def visit_constant(self, _, visited_children):
        return Constant(visited_children[0])

    def visit_variable(self, node, _):
        return Variable(node.full_text[node.start:node.end])

    def visit_term(self, _, visited_children):
        return visited_children[0]

    def visit_term_list(self, _, visited_children):
        if len(visited_children[1]) == 0:
            return visited_children[:1]
        else:
            head, ((_, _, tail),), _ = visited_children
            return [head] + tail

    def visit_atom_list(self, _, visited_children):
        if len(visited_children[1]) == 0:
            return visited_children[:1]
        else:
            head, ((_, _, tail),), _ = visited_children
            return [head] + tail

    def visit_atom(self, _, visited_children):
        return Atom(visited_children[1], *visited_children[4], negated=len(visited_children[0]) == 1)

    def visit_low_id(self, node, _):
        return node.full_text[node.start:node.end]

    def visit_up_id(self, node, _):
        return node.full_text[node.start:node.end]

    def visit_string(self, node, _):
        return node.full_text[node.start:node.end]

    def generic_visit(self, node, visited_children):
        return visited_children

    def visit_neg(self, _1, _2):
        return True
<commit_msg>edit to @riedelcastro's parser<commit_after># -*- coding: utf-8 -*-

from parsimonious.grammar import Grammar, NodeVisitor

grammar = Grammar(
    """
    clause      = atom ( ":-" _ atom_list)?
    atom_list   = atom  ("," _ atom_list)? _
    atom        = neg? predicate "(" _ term_list ")" _
    neg         = "!" _
    term_list   = term  ("," _ term_list)? _
    term        = constant / variable
    predicate   = low_high_id / string
    constant    = low_id / string
    variable    = ~"[A-Z][a-z A-Z 0-9_]*"
    low_high_id = ~"[a-zA-Z_./][a-z A-Z 0-9_./]*"
    low_id      = ~"[a-z_./][a-z A-Z 0-9_./]*"
    string      = ~r"'[^']*'" / ~r"\\"[^\\"]*\\""
    _           = skip*
    skip        = ~r"\s+"
    """)

class Expr:
    def __repr__(self):
        return self.__dict__.__repr__()

    def __eq__(self, other):
        return isinstance(other, self.__class__) and str(other) == str(self)

    def __hash__(self):
        return self.__str__().__hash__()

class Variable(Expr):
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Constant(Expr):
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return self.name

class Predicate(Expr):
    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        return isinstance(other, self.__class__) and other.name == self.name

    def __hash__(self):
        return self.name.__hash__()

    def __repr__(self):
        return self.name

class Atom(Expr):
    def __init__(self, predicate: Predicate, *arguments, negated=False):
        self.predicate = predicate
        self.arguments = arguments
        self.negated = negated

    def __repr__(self):
        return "{}{}({})".format("!" if self.negated else "", self.predicate.name, ", ".join(str(x) for x in self.arguments))

class Clause(Expr):
    def __init__(self, head: Atom, *body):
        self.head = head
        self.body = body

    def __repr__(self):
        return str(self.head) if len(self.body) == 0 else "{head} :- {body}".format(
            head=self.head,
            body=", ".join(str(x) for x in self.body))

class ClauseVisitor(NodeVisitor):
    def visit_clause(self, node, visited_children):
        if len(visited_children[1]) == 0:
            return Clause(visited_children[0])
        else:
            head, ((_, _, body),) = visited_children
            return Clause(head, *body)

    def visit_predicate(self, _, visited_children):
        return Predicate(visited_children[0])

    def visit_constant(self, _, visited_children):
        return Constant(visited_children[0])

    def visit_variable(self, node, _):
        return Variable(node.full_text[node.start:node.end])

    def visit_term(self, _, visited_children):
        return visited_children[0]

    def visit_term_list(self, _, visited_children):
        if len(visited_children[1]) == 0:
            return visited_children[:1]
        else:
            head, ((_, _, tail),), _ = visited_children
            return [head] + tail

    def visit_atom_list(self, _, visited_children):
        if len(visited_children[1]) == 0:
            return visited_children[:1]
        else:
            head, ((_, _, tail),), _ = visited_children
            return [head] + tail

    def visit_atom(self, _, visited_children):
        return Atom(visited_children[1], *visited_children[4], negated=len(visited_children[0]) == 1)

    def visit_low_id(self, node, _):
        return node.full_text[node.start:node.end]

    def visit_up_id(self, node, _):
        return node.full_text[node.start:node.end]

    def visit_string(self, node, _):
        return node.full_text[node.start:node.end]

    def generic_visit(self, node, visited_children):
        return visited_children

    def visit_neg(self, _1, _2):
        return True
<|endoftext|>