<commit_before># Copyright 2013 Mario Graff Guerrero

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from SimpleGP.forest import SubTreeXO
from SimpleGP.sparse_array import SparseEval, SparseArray
from SimpleGP.Simplify import Simplify
import tempfile
import os

class SparseGPPG(SubTreeXO):
    def __init__(self, prototypes=[], tree_cl=None,
                 **kwargs):
        super(SparseGPPG, self).__init__(**kwargs)
        self._prototypes = prototypes
        if tree_cl is not None:
            self._tree_cl = np.array(tree_cl)
        else:
            self._tree_cl = tree_cl
        self._pg_cl = None
        self._pg_d = None

    @property
    def prototypes(self):
        p = (self.population[self.best],
             self._p_constants[self.best],
             self._tree_cl)
        return self._prototypes + [p]

    def save_extras(self, fpt):
        super(SparseGPPG, self).save_extras(fpt)
        np.save(fpt, self._prototypes)
        np.save(fpt, self._tree_cl)

    def load_extras(self, fpt):
        super(SparseGPPG, self).load_extras(fpt)
        self._prototypes += np.load(fpt).tolist()
        self._tree_cl = np.load(fpt)
        self._ntrees = self._tree_cl.shape[0]
        self.init_ntrees()
        self.eval_prev_prototypes()

    def associate_terminals_cl(self):
        self._type_terminals = np.zeros(self._ntrees,
                                        dtype=np.object)
        for k, v in enumerate(self._tree_cl):
            self._type_terminals[k] = np.where(self._f == v)[0]

    def init_ntrees(self):
        cdn = "ntress must be greater or equal to the number of classes"
        if self._ntrees is None and self._tree_cl is None:
            cl = np.unique(self._f)
            self._ntrees = cl.shape[0]
            self._tree_cl = np.array(map(lambda x: x % cl.shape[0],
                                         range(self._ntrees)))
        elif self._tree_cl is not None:
            self._ntrees = len(self._tree_cl)
        elif self._ntrees is not None:
            cl = np.unique(self._f)
            if cl.shape[0] > self._ntrees:
                raise NotImplementedError(cdn)
            self._tree_cl = np.array(map(lambda x: x % cl.shape[0],
                                         range(self._ntrees)))
        if len(self._tree_cl) != self._ntrees:
            raise NotImplementedError(cdn)
        self._nop[self._output_pos] = self._ntrees
        self._ntrees_per_class_mod = self._ntrees
        self.associate_terminals_cl()
        self._dist_matrix = np.zeros((self._ntrees, self._nvar))

    def train(self, x, f):
        self._eval = SparseEval(self._nop)
        self._eval.X(x)
        self._nvar = len(x)
        self._x = x
        self._f = f
        self._st = None
        self._p_der_st = None
        self._error_st = None
        self._simplify = Simplify(self.nvar, self._nop)
        self._simplify.set_constants(self._constants2)
        self._tree.set_nvar(self.nvar)
        self.init_ntrees()
        self.eval_prev_prototypes()
        return self

    def random_leaf(self):
        """Choose a leaf depending on the i-th argument of root, i.e.,
        Output function  """
        if np.random.rand() < 0.5 or self._nrandom == 0:
            i = self._doing_tree
            cnt = self._type_terminals[i].shape[0]
            l = np.random.randint(cnt)
            l = self._type_terminals[i][l]
            return l + self.nfunc
        else:
            l = np.random.randint(self._constants.shape[0])
            return l + self.nfunc + self.nvar

    @property
    def nvar(self):
        """Number of independent variables"""
        return self._nvar

    def create_population(self):
        flag = super(SparseGPPG, self).create_population()
        self.eval_prev_prototypes()
        return flag

    def eval_prev_prototypes(self):
        l, cl = self.eval_prototypes(self._prototypes)
        if len(l):
            tree_cl = cl
            D = np.zeros((len(l), len(self._x)))
            SparseArray.distance(self._x, l, D)
            s = D.argmin(axis=0)
            self._pg_cl = tree_cl[s]
            self._pg_d = D[s, np.arange(s.shape[0])]
        else:
            self._pg_cl = None
            self._pg_d = None

    def eval_prototypes(self, prototypes):
        if len(prototypes) == 0:
            return [], []
        l = []
        cl = []
        for ind, c, tree_cl in prototypes:
            self._nop[self._output_pos] = tree_cl.shape[0]
            self._eval.eval(ind, c, to_np_array=False)
            l += self._eval.get_output()
            cl.append(tree_cl)
        self._nop[self._output_pos] = self._ntrees
        return l, np.concatenate(cl)

    def prototypes_performance(self):
        l, cl = self.eval_prototypes(self.prototypes)
        D = np.zeros((len(l), len(self._x)))
        SparseArray.distance(self._x, l, D)
        s = D.argmin(axis=0)
        perf = np.zeros(len(l))
        for k, v in zip(s, cl[s] == self._f):
            perf[k] += v
        s = np.array(map(lambda x: (s == x).sum(), range(len(l))))
        return s, perf

    def eval_ind(self, ind, pos=0, constants=None):
        c = constants if constants is not None else self._constants
        self.nodes_evaluated += ind.shape[0]
        self._eval.eval(ind, c, to_np_array=False)
        yh = self._eval.get_output()
        SparseArray.distance(self._x, yh, self._dist_matrix)
        mn_finite = ~np.isfinite(self._dist_matrix)
        self._dist_matrix[mn_finite] = np.inf
        s = self._dist_matrix.argmin(axis=0)
        yh = self._tree_cl[s]
        if self._pg_d is not None:
            d = self._dist_matrix[s, np.arange(s.shape[0])]
            W = np.vstack((self._pg_d, d))
            s = W.argmin(axis=0)
            yh = np.vstack((self._pg_cl, yh))[s, np.arange(s.shape[0])]
        return yh

    def predict(self, X, ind=None, nprototypes=None):
        if ind is None:
            ind = self.best
        prototypes = self.prototypes
        npro = nprototypes if nprototypes is not None else len(prototypes)
        l, cl = self.eval_prototypes(self.prototypes[:npro])
        D = np.zeros((len(l), len(X)))
        SparseArray.distance(X, l, D)
        s = D.argmin(axis=0)
        return cl[s]

    def distance(self, _y, _yh):
        return -self.recall(_y, _yh).mean()

    def verbose(self, i):
        if self._verbose:
            p = self.precision(self._f, self.eval())
            r = self.recall(self._f, self.eval())
            print "Iter:", i, "R:", map(lambda x: "%0.4f" % x, r),\
                "P:", map(lambda x: "%0.4f" % x, p)

    @classmethod
    def run_cl(cls, X, y, nprototypes=10, fname_best=None,
               func=['+', '-', 'abs', 'sin', 'sq', 'sqrt', 'sigmoid', 'if'],
               nrandom=0, verbose=False, max_length=512, tree_cl=None,
               seed=0, prototypes=None, **kwargs):
        prototypes = [] if prototypes is None else prototypes
        nprot = 0
        fbest = -np.inf
        fname = None
        while nprot < nprototypes:
            if fname_best is not None:
                if fname is None:
                    fname = fname_best
                else:
                    fname = tempfile.mktemp() + ".npy"
                    if fname_best.count('.gz'):
                        fname += '.gz'
            gp = cls(fname_best=fname, prototypes=prototypes,
                     tree_cl=tree_cl, nrandom=nrandom, verbose=verbose,
                     func=func, max_length=max_length,
                     seed=seed, **kwargs).train(X, y)
            gp.run()
            nprot = len(gp.prototypes)
            if fname_best is not None and fname != fname_best:
                try:
                    os.unlink(fname_best)
                except OSError:
                    pass
                os.link(fname, fname_best)
                os.unlink(fname)
            gp.verbose(nprot)
            if fbest > -np.inf and fbest > gp.fitness(gp.best):
                return gp
            else:
                fbest = gp.fitness(gp.best)
                prototypes = gp.prototypes
                tree_cl = [gp.recall(gp._f, gp.eval()).argmin()]
        return gp

    @classmethod
    def f1(cls, y, yh):
        p = cls.precision(y, yh)
        r = cls.recall(y, yh)
        f1 = 2 * p * r / (p + r)
        f1[~np.isfinite(f1)] = -np.inf
        return f1

    @staticmethod
    def recall(y, yh):
        l = []
        for cl in np.unique(y):
            m = y == cl
            r = (yh[m] == cl).sum() / float(m.sum())
            l.append(r)
        return np.array(l)

    @staticmethod
    def precision(y, yh):
        l = []
        for cl in np.unique(y):
            m = yh == cl
            p = (y[m] == cl).sum() / float(m.sum())
            l.append(p)
        return np.array(l)
<commit_msg>Minor bug in predict<commit_after># Copyright 2013 Mario Graff Guerrero

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from SimpleGP.forest import SubTreeXO
from SimpleGP.sparse_array import SparseEval, SparseArray
from SimpleGP.Simplify import Simplify
import tempfile
import os

class SparseGPPG(SubTreeXO):
    def __init__(self, prototypes=[], tree_cl=None,
                 **kwargs):
        super(SparseGPPG, self).__init__(**kwargs)
        self._prototypes = prototypes
        if tree_cl is not None:
            self._tree_cl = np.array(tree_cl)
        else:
            self._tree_cl = tree_cl
        self._pg_cl = None
        self._pg_d = None

    @property
    def prototypes(self):
        p = (self.population[self.best],
             self._p_constants[self.best],
             self._tree_cl)
        return self._prototypes + [p]

    def save_extras(self, fpt):
        super(SparseGPPG, self).save_extras(fpt)
        np.save(fpt, self._prototypes)
        np.save(fpt, self._tree_cl)

    def load_extras(self, fpt):
        super(SparseGPPG, self).load_extras(fpt)
        self._prototypes += np.load(fpt).tolist()
        self._tree_cl = np.load(fpt)
        self._ntrees = self._tree_cl.shape[0]
        self.init_ntrees()
        self.eval_prev_prototypes()

    def associate_terminals_cl(self):
        self._type_terminals = np.zeros(self._ntrees,
                                        dtype=np.object)
        for k, v in enumerate(self._tree_cl):
            self._type_terminals[k] = np.where(self._f == v)[0]

    def init_ntrees(self):
        cdn = "ntress must be greater or equal to the number of classes"
        if self._ntrees is None and self._tree_cl is None:
            cl = np.unique(self._f)
            self._ntrees = cl.shape[0]
            self._tree_cl = np.array(map(lambda x: x % cl.shape[0],
                                         range(self._ntrees)))
        elif self._tree_cl is not None:
            self._ntrees = len(self._tree_cl)
        elif self._ntrees is not None:
            cl = np.unique(self._f)
            if cl.shape[0] > self._ntrees:
                raise NotImplementedError(cdn)
            self._tree_cl = np.array(map(lambda x: x % cl.shape[0],
                                         range(self._ntrees)))
        if len(self._tree_cl) != self._ntrees:
            raise NotImplementedError(cdn)
        self._nop[self._output_pos] = self._ntrees
        self._ntrees_per_class_mod = self._ntrees
        self.associate_terminals_cl()
        self._dist_matrix = np.zeros((self._ntrees, self._nvar))

    def train(self, x, f):
        self._eval = SparseEval(self._nop)
        self._eval.X(x)
        self._nvar = len(x)
        self._x = x
        self._f = f
        self._st = None
        self._p_der_st = None
        self._error_st = None
        self._simplify = Simplify(self.nvar, self._nop)
        self._simplify.set_constants(self._constants2)
        self._tree.set_nvar(self.nvar)
        self.init_ntrees()
        self.eval_prev_prototypes()
        return self

    def random_leaf(self):
        """Choose a leaf depending on the i-th argument of root, i.e.,
        Output function  """
        if np.random.rand() < 0.5 or self._nrandom == 0:
            i = self._doing_tree
            cnt = self._type_terminals[i].shape[0]
            l = np.random.randint(cnt)
            l = self._type_terminals[i][l]
            return l + self.nfunc
        else:
            l = np.random.randint(self._constants.shape[0])
            return l + self.nfunc + self.nvar

    @property
    def nvar(self):
        """Number of independent variables"""
        return self._nvar

    def create_population(self):
        flag = super(SparseGPPG, self).create_population()
        self.eval_prev_prototypes()
        return flag

    def eval_prev_prototypes(self):
        l, cl = self.eval_prototypes(self._prototypes)
        if len(l):
            tree_cl = cl
            D = np.zeros((len(l), len(self._x)))
            SparseArray.distance(self._x, l, D)
            s = D.argmin(axis=0)
            self._pg_cl = tree_cl[s]
            self._pg_d = D[s, np.arange(s.shape[0])]
        else:
            self._pg_cl = None
            self._pg_d = None

    def eval_prototypes(self, prototypes):
        if len(prototypes) == 0:
            return [], []
        l = []
        cl = []
        for ind, c, tree_cl in prototypes:
            self._nop[self._output_pos] = tree_cl.shape[0]
            self._eval.eval(ind, c, to_np_array=False)
            l += self._eval.get_output()
            cl.append(tree_cl)
        self._nop[self._output_pos] = self._ntrees
        return l, np.concatenate(cl)

    def prototypes_performance(self):
        l, cl = self.eval_prototypes(self.prototypes)
        D = np.zeros((len(l), len(self._x)))
        SparseArray.distance(self._x, l, D)
        s = D.argmin(axis=0)
        perf = np.zeros(len(l))
        for k, v in zip(s, cl[s] == self._f):
            perf[k] += v
        s = np.array(map(lambda x: (s == x).sum(), range(len(l))))
        return s, perf

    def eval_ind(self, ind, pos=0, constants=None):
        c = constants if constants is not None else self._constants
        self.nodes_evaluated += ind.shape[0]
        self._eval.eval(ind, c, to_np_array=False)
        yh = self._eval.get_output()
        SparseArray.distance(self._x, yh, self._dist_matrix)
        mn_finite = ~np.isfinite(self._dist_matrix)
        self._dist_matrix[mn_finite] = np.inf
        s = self._dist_matrix.argmin(axis=0)
        yh = self._tree_cl[s]
        if self._pg_d is not None:
            d = self._dist_matrix[s, np.arange(s.shape[0])]
            W = np.vstack((self._pg_d, d))
            s = W.argmin(axis=0)
            yh = np.vstack((self._pg_cl, yh))[s, np.arange(s.shape[0])]
        return yh

    def predict(self, X, ind=None, nprototypes=None, prototypes=None):
        if ind is None:
            ind = self.best
        p = (self.population[ind],
             self._p_constants[ind],
             self._tree_cl)
        if prototypes is None:
            prototypes = self._prototypes + [p]
        npro = nprototypes if nprototypes is not None else len(prototypes)
        l, cl = self.eval_prototypes(self.prototypes[:npro])
        D = np.zeros((len(l), len(X)))
        SparseArray.distance(X, l, D)
        s = D.argmin(axis=0)
        return cl[s]

    def distance(self, _y, _yh):
        return -self.recall(_y, _yh).mean()

    def verbose(self, i):
        if self._verbose:
            p = self.precision(self._f, self.eval())
            r = self.recall(self._f, self.eval())
            print "Iter:", i, "R:", map(lambda x: "%0.4f" % x, r),\
                "P:", map(lambda x: "%0.4f" % x, p)

    @classmethod
    def run_cl(cls, X, y, nprototypes=10, fname_best=None,
               func=['+', '-', 'abs', 'sin', 'sq', 'sqrt', 'sigmoid', 'if'],
               nrandom=0, verbose=False, max_length=512, tree_cl=None,
               seed=0, prototypes=None, **kwargs):
        prototypes = [] if prototypes is None else prototypes
        nprot = 0
        fbest = -np.inf
        fname = None
        while nprot < nprototypes:
            if fname_best is not None:
                if fname is None:
                    fname = fname_best
                else:
                    fname = tempfile.mktemp() + ".npy"
                    if fname_best.count('.gz'):
                        fname += '.gz'
            gp = cls(fname_best=fname, prototypes=prototypes,
                     tree_cl=tree_cl, nrandom=nrandom, verbose=verbose,
                     func=func, max_length=max_length,
                     seed=seed, **kwargs).train(X, y)
            gp.run()
            nprot = len(gp.prototypes)
            if fname_best is not None and fname != fname_best:
                try:
                    os.unlink(fname_best)
                except OSError:
                    pass
                os.link(fname, fname_best)
                os.unlink(fname)
            gp.verbose(nprot)
            if fbest > -np.inf and fbest > gp.fitness(gp.best):
                return gp
            else:
                fbest = gp.fitness(gp.best)
                prototypes = gp.prototypes
                tree_cl = [gp.recall(gp._f, gp.eval()).argmin()]
        return gp

    @classmethod
    def f1(cls, y, yh):
        p = cls.precision(y, yh)
        r = cls.recall(y, yh)
        f1 = 2 * p * r / (p + r)
        f1[~np.isfinite(f1)] = -np.inf
        return f1

    @staticmethod
    def recall(y, yh):
        l = []
        for cl in np.unique(y):
            m = y == cl
            r = (yh[m] == cl).sum() / float(m.sum())
            l.append(r)
        return np.array(l)

    @staticmethod
    def precision(y, yh):
        l = []
        for cl in np.unique(y):
            m = yh == cl
            p = (y[m] == cl).sum() / float(m.sum())
            l.append(p)
        return np.array(l)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-:
from django.contrib.auth.models import User
from django.contrib.comments import get_model
from django.core import urlresolvers
from django.core.cache import cache
from django.db import models
from django.shortcuts import get_object_or_404
from resrc.utils import slugify

from taggit.managers import TaggableManager

LEVELS = ['beginner', 'intermediate', 'advanced']

class LinkManager(models.Manager):

    def latest(self,limit=10, lang_filter=[1]):
        qs = self.get_query_set()
        if lang_filter:
            qs = qs.filter(language__in=lang_filter)
        qs = qs.values('pk', 'slug', 'title') \
               .order_by('-pubdate')[:limit]
        return qs

class Link(models.Model):

    class Meta:
        verbose_name = 'Link'
        verbose_name_plural = 'Links'

    objects = LinkManager()

    title = models.CharField('title', max_length=120)

    slug = models.SlugField(max_length=255)

    hash2 = models.CharField(max_length=11, editable=False)

    url = models.URLField('url')

    pubdate = models.DateTimeField(
        'date added', auto_now_add=True, editable=False)

    author = models.ForeignKey(User, verbose_name='author')

    level = models.CharField('Level', max_length=30, choices=zip(LEVELS, LEVELS), null=True, blank=True)

    language = models.ForeignKey('language.Language', default=1)

    tags = TaggableManager()

    def save(self, *args, **kwargs):
        cache.delete('link_%s' % self.pk)
        self.do_unique_slug()
        if not self.id:
            from resrc.utils.hash2 import hash2
            self.hash2 = hash2(self.url)
        cache.set('link_%s' % self.pk, self, 60*5)
        super(Link, self).save(*args, **kwargs)

    def do_unique_slug(self):
        """
        Ensures that the slug is always unique for this post
        """
        if not self.id:
            # make sure we have a slug first
            if not len(self.slug.strip()):
                self.slug = slugify(self.title)

            self.slug = self.get_unique_slug(self.slug)
            return True

        return False

    def get_unique_slug(self, slug):
        """
        Iterates until a unique slug is found
        """
        orig_slug = slug
        counter = 1

        while True:
            links = Link.objects.filter(slug=slug)
            if not links.exists():
                return slug

            slug = '%s-%s' % (orig_slug, counter)
            counter += 1

    def __unicode__(self):
        return self.title

    def get_comment_count(self):
        return get_model().objects.filter(object_pk=self.pk).count()

    def get_absolute_url(self):
        return urlresolvers.reverse("link-single-slug", args=(
            self.pk,
            self.slug
        ))

    def get_lang(self):
        from django.conf import settings
        return [x[1] for x in settings.LANGUAGES if x[0] == self.language.language][0]

    def vote(self, user, list_pk=None):
        from resrc.vote.models import Vote
        # we keep track of list because if link is voted from a list, we also vote for the list
        if list_pk is not None:
            from resrc.list.models import List
            alist = get_object_or_404(List, pk=list_pk)
        else:
            alist = None
        vote = Vote.objects.create(
            user=user,
            link=self,
            alist=alist
        )
        vote.save()

    def get_votes(self):
        from resrc.vote.models import Vote
        return Vote.objects.filter(link=self).count()

    def get_categories(self):
        categories = ['book', 'tutorial', 'interactive tutorial', 'guide',
            'resource', 'cheat sheet', 'article', 'blog', 'reference',
            'documentation', 'course', 'slides', 'video', 'MOOC']
        cats = []
        for tag in self.tags.all().values_list('name', flat=True):
            if tag in categories:
                cats.append(tag)
        if cats:
            return "(%s)" % "/".join(cats)
        else:
            return ""

class RevisedLink(models.Model):
    link = models.ForeignKey('link.Link')
    title = models.CharField('title', max_length=120, null=True, blank=True)
    url = models.URLField('url', null=True, blank=True)
    level = models.CharField('Level', max_length=30, choices=zip(LEVELS, LEVELS), null=True, blank=True)
    language = models.ForeignKey('language.Language', null=True, blank=True)
    tags = models.CharField(max_length=255, null=True, blank=True)

    def __unicode__(self):
        return "link %d - %s" % (self.link.pk, self.link.title)
<commit_msg>Add talk to link categories<commit_after># -*- coding: utf-8 -*-:
from django.contrib.auth.models import User
from django.contrib.comments import get_model
from django.core import urlresolvers
from django.core.cache import cache
from django.db import models
from django.shortcuts import get_object_or_404
from resrc.utils import slugify

from taggit.managers import TaggableManager

LEVELS = ['beginner', 'intermediate', 'advanced']

class LinkManager(models.Manager):

    def latest(self,limit=10, lang_filter=[1]):
        qs = self.get_query_set()
        if lang_filter:
            qs = qs.filter(language__in=lang_filter)
        qs = qs.values('pk', 'slug', 'title') \
               .order_by('-pubdate')[:limit]
        return qs

class Link(models.Model):

    class Meta:
        verbose_name = 'Link'
        verbose_name_plural = 'Links'

    objects = LinkManager()

    title = models.CharField('title', max_length=120)

    slug = models.SlugField(max_length=255)

    hash2 = models.CharField(max_length=11, editable=False)

    url = models.URLField('url')

    pubdate = models.DateTimeField(
        'date added', auto_now_add=True, editable=False)

    author = models.ForeignKey(User, verbose_name='author')

    level = models.CharField('Level', max_length=30, choices=zip(LEVELS, LEVELS), null=True, blank=True)

    language = models.ForeignKey('language.Language', default=1)

    tags = TaggableManager()

    def save(self, *args, **kwargs):
        cache.delete('link_%s' % self.pk)
        self.do_unique_slug()
        if not self.id:
            from resrc.utils.hash2 import hash2
            self.hash2 = hash2(self.url)
        cache.set('link_%s' % self.pk, self, 60*5)
        super(Link, self).save(*args, **kwargs)

    def do_unique_slug(self):
        """
        Ensures that the slug is always unique for this post
        """
        if not self.id:
            # make sure we have a slug first
            if not len(self.slug.strip()):
                self.slug = slugify(self.title)

            self.slug = self.get_unique_slug(self.slug)
            return True

        return False

    def get_unique_slug(self, slug):
        """
        Iterates until a unique slug is found
        """
        orig_slug = slug
        counter = 1

        while True:
            links = Link.objects.filter(slug=slug)
            if not links.exists():
                return slug

            slug = '%s-%s' % (orig_slug, counter)
            counter += 1

    def __unicode__(self):
        return self.title

    def get_comment_count(self):
        return get_model().objects.filter(object_pk=self.pk).count()

    def get_absolute_url(self):
        return urlresolvers.reverse("link-single-slug", args=(
            self.pk,
            self.slug
        ))

    def get_lang(self):
        from django.conf import settings
        return [x[1] for x in settings.LANGUAGES if x[0] == self.language.language][0]

    def vote(self, user, list_pk=None):
        from resrc.vote.models import Vote
        # we keep track of list because if link is voted from a list, we also vote for the list
        if list_pk is not None:
            from resrc.list.models import List
            alist = get_object_or_404(List, pk=list_pk)
        else:
            alist = None
        vote = Vote.objects.create(
            user=user,
            link=self,
            alist=alist
        )
        vote.save()

    def get_votes(self):
        from resrc.vote.models import Vote
        return Vote.objects.filter(link=self).count()

    def get_categories(self):
        categories = ['book', 'tutorial', 'interactive tutorial', 'guide',
            'resource', 'cheat sheet', 'article', 'blog', 'reference',
            'documentation', 'course', 'slides', 'video', 'MOOC', 'talk']
        cats = []
        for tag in self.tags.all().values_list('name', flat=True):
            if tag in categories:
                cats.append(tag)
        if cats:
            return "(%s)" % "/".join(cats)
        else:
            return ""

class RevisedLink(models.Model):
    link = models.ForeignKey('link.Link')
    title = models.CharField('title', max_length=120, null=True, blank=True)
    url = models.URLField('url', null=True, blank=True)
    level = models.CharField('Level', max_length=30, choices=zip(LEVELS, LEVELS), null=True, blank=True)
    language = models.ForeignKey('language.Language', null=True, blank=True)
    tags = models.CharField(max_length=255, null=True, blank=True)

    def __unicode__(self):
        return "link %d - %s" % (self.link.pk, self.link.title)
<|endoftext|>


<commit_before>from rest_framework import permissions, viewsets, status
from rest_framework.decorators import detail_route, list_route, parser_classes
from rest_framework.response import Response
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import BaseParser

from retail.sale.models import Sale, SaleItem, Tender, TenderType, SalesChannel, Purchaser
from retail.sale.serializers import SaleSerializer, SaleItemSerializer, TenderSerializer, TenderTypeSerializer, SalesChannelSerializer, PurchaserSerializer
from retail.sale.utils import ImportDSRLine, ImportDSR

class PlainTextParser(BaseParser):
    """
    Plain text parser.
    """
    media_type = 'text/plain'

    def parse(self, stream, media_type=None, parser_context=None):
        """
        Simply return a string representing the body of the request.
        """
        return stream.read()

class SaleViewSet(viewsets.ModelViewSet):
    queryset = Sale.objects.all()
    serializer_class = SaleSerializer
    permission_classes = (permissions.DjangoModelPermissions, )
    filter_fields = ('channel','store','vendor',)

    @list_route(methods=['post'], parser_classes=(PlainTextParser,))
    def dsr_single(self, request):
        try:
            sale = ImportDSRLine(request.data.decode("utf-8", "ignore"))
            serializer = self.get_serializer(sale)
            return Response(serializer.data)
        except:
            return Response({'Error':'foo'},
                            status=status.HTTP_400_BAD_REQUEST)

    @list_route(methods=['post'], parser_classes=(PlainTextParser,))
    def dsr(self, request):
        try:
            sale = ImportDSR(request.data.decode("utf-8", "ignore"))
            return Response("Created {} sales".format(sale[5]))
        except:
            return Response({'Error':'foo'},
                            status=status.HTTP_400_BAD_REQUEST)

class SalesChannelViewSet(viewsets.ModelViewSet):
    queryset = SalesChannel.objects.all()
    serializer_class = SalesChannelSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class SaleItemViewSet(viewsets.ModelViewSet):
    queryset = SaleItem.objects.all()
    serializer_class = SaleItemSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class TenderViewSet(viewsets.ModelViewSet):
    queryset = Tender.objects.all()
    serializer_class = TenderSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class TenderTypeViewSet(viewsets.ModelViewSet):
    queryset = TenderType.objects.all()
    serializer_class = TenderTypeSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class PurchaserViewSet(viewsets.ModelViewSet):
    queryset = Purchaser.objects.all()
    serializer_class = PurchaserSerializer
    permission_classes = (permissions.DjangoModelPermissions, )
    <commit_msg>Removed tests for utils<commit_after>from rest_framework import permissions, viewsets, status
from rest_framework.decorators import detail_route, list_route, parser_classes
from rest_framework.response import Response
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import BaseParser

from retail.sale.models import Sale, SaleItem, Tender, TenderType, SalesChannel, Purchaser
from retail.sale.serializers import SaleSerializer, SaleItemSerializer, TenderSerializer, TenderTypeSerializer, SalesChannelSerializer, PurchaserSerializer

class PlainTextParser(BaseParser):
    """
    Plain text parser.
    """
    media_type = 'text/plain'

    def parse(self, stream, media_type=None, parser_context=None):
        """
        Simply return a string representing the body of the request.
        """
        return stream.read()

class SaleViewSet(viewsets.ModelViewSet):
    queryset = Sale.objects.all()
    serializer_class = SaleSerializer
    permission_classes = (permissions.DjangoModelPermissions, )
    filter_fields = ('channel','store','vendor',)

class SalesChannelViewSet(viewsets.ModelViewSet):
    queryset = SalesChannel.objects.all()
    serializer_class = SalesChannelSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class SaleItemViewSet(viewsets.ModelViewSet):
    queryset = SaleItem.objects.all()
    serializer_class = SaleItemSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class TenderViewSet(viewsets.ModelViewSet):
    queryset = Tender.objects.all()
    serializer_class = TenderSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class TenderTypeViewSet(viewsets.ModelViewSet):
    queryset = TenderType.objects.all()
    serializer_class = TenderTypeSerializer
    permission_classes = (permissions.DjangoModelPermissions, )

class PurchaserViewSet(viewsets.ModelViewSet):
    queryset = Purchaser.objects.all()
    serializer_class = PurchaserSerializer
    permission_classes = (permissions.DjangoModelPermissions, )
    <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    revision
    ~~~~~~~~

    :copyright: (c) 2017 by COLORFUL BOARD Inc.
    :license: MIT, see LICENSE for more details.
"""

from __future__ import absolute_import

__version__ = "0.0.3"
__license__ = "MIT"
<commit_msg>chore(source): bump to dev version<commit_after># -*- coding: utf-8 -*-
"""
    revision
    ~~~~~~~~

    :copyright: (c) 2017 by COLORFUL BOARD Inc.
    :license: MIT, see LICENSE for more details.
"""

from __future__ import absolute_import

__version__ = "0.0.4"
__license__ = "MIT"
<|endoftext|>


<commit_before># Copyright 2012 the rootpy developers
# distributed under the terms of the GNU General Public License
# Original author: Scott Snyder scott.snyder(a)cern.ch, 2004.

"""Pickle python data into a ROOT file, preserving references to ROOT objects.

This module allows pickling python objects into a ROOT file. The python
objects may contain references to named ROOT objects. If one has set up a
structure of python objects to hold ROOT histograms, this provides a convenient
way of saving and restoring your histograms. The pickled python data are
stored in an additional string object in the ROOT file; any ROOT objects are
stored as usual. (Thus, ROOT files written by the pickler can be read just
like any other ROOT file if you don't care about the python data.)

Here's an example of writing a pickle::

   from rootpy.plotting import Hist
   from rootpy.io.pickler import dump
   hlist = []
   for i in range(10):
       hlist.append(Hist(10, 0, 10, name='h{0}'.format(i)))
   dump(hlist, 'test.root')

This writes a list of histograms to test.root. The histograms may be read back
like this::

   from rootpy.io.pickler import load
   hlist = load('test.root')

The following additional notes apply:

* Pickling may not always work correctly for the case of python objects
  deriving from ROOT objects. It will probably also not work for the case of
  ROOT objects which do not derive from TObject.

* When the pickled data are being read, if a class doesn't exist,
  a dummy class with no methods will be used instead. This is different
  from the standard pickle behavior (where it would be an error), but it
  simplifies usage in the common case where the class is being used to hold
  histograms, and its methods are entirely concerned with filling the
  histograms.

* When restoring a reference to a ROOT object, the default behavior
  is to not read the ROOT object itself, but instead to create a proxy. The
  ROOT object will then be read the first time the proxy is accessed. This can
  help significantly with time and memory usage if you're only accessing a
  small fraction of the ROOT objects, but it does mean that you need to keep
  the ROOT file open. Pass use_proxy=0 to disable this behavior.

"""
from . import log; log = log[__name__]
from . import root_open
from ..context import preserve_current_directory

from cStringIO import StringIO
import cPickle
import ROOT
import sys

__all__ = [
    'dump',
    'load',
    'compat_hooks',
]

_compat_hooks = None
xdict = {}
xserial = 0

# Argh!  We can't store NULs in TObjStrings.
# But pickle protocols > 0 are binary protocols, and will get corrupted
# if we truncate at a NUL.
# So, when we save the pickle data, make the mappings:
#  0x00 -> 0xff 0x01
#  0xff -> 0xff 0xfe
def _protect(s):
    return s.replace('\377', '\377\376').replace('\000', '\377\001')
def _restore(s):
    return s.replace('\377\001', '\000').replace('\377\376', '\377')

class IO_Wrapper:
    def __init__(self):
        return self.reopen()

    def write(self, s):
        return self.__s.write(_protect(s))

    def read(self, i):
        return self.__s.read(i)

    def readline(self):
        return self.__s.readline()

    def getvalue(self):
        return self.__s.getvalue()

    def setvalue(self, s):
        self.__s = StringIO(_restore(s))
        return

    def reopen(self):
        self.__s = StringIO()
        return

class Pickler:
    def __init__(self, file, proto=0):
        """Create a root pickler.
        `file` should be a ROOT TFile. `proto` is the python pickle protocol
        version to use.  The python part will be pickled to a ROOT
        TObjString called _pickle; it will contain references to the
        ROOT objects.
        """
        self.__file = file
        self.__keys = file.GetListOfKeys()
        self.__io = IO_Wrapper()
        self.__pickle = cPickle.Pickler(self.__io, proto)
        self.__pickle.persistent_id = self._persistent_id
        self.__pmap = {}

    def dump(self, o, key=None):
        """Write a pickled representation of o to the open TFile."""
        if key == None:
            key = '_pickle'
        with preserve_current_directory():
            self.__file.cd()
            self.__pickle.dump(o)
            s = ROOT.TObjString(self.__io.getvalue())
            self.__io.reopen()
            s.Write(key)
            self.__file.GetFile().Flush()
            self.__pmap.clear()

    def clear_memo(self):
        """Clears the pickler's internal memo."""
        self.__pickle.memo.clear()

    def _persistent_id(self, o):
        if hasattr(o, '_ROOT_Proxy__obj'):
            o = o._ROOT_Proxy__obj()
        if isinstance(o, ROOT.TObject):
            # Write the object, and return the resulting NAME;CYCLE.
            # We used to to this like this:
            #o.Write()
            #k = self.__file.GetKey (o.GetName())
            #pid = "%s;%d" % (k.GetName(), k.GetCycle())
            # It turns out, though, that destroying the python objects
            # referencing the TKeys is quite expensive (O(logN) where
            # N is the total number of pyroot objects?).  Although
            # we want to allow for the case of saving multiple objects
            # with the same name, the most common case is that the name
            # has not already been written to the file.  So we optimize
            # for that case, doing the key lookup before we write the
            # object, not after.  (Note further: GetKey() is very slow
            # if the key does not actually exist, as it does a linear
            # search of the key list.  We use FindObject instead for the
            # initial lookup, which is a hashed lookup, but it is not
            # guaranteed to find the highest cycle.  So if we do
            # find an existing key, we need to look up again using GetKey.
            nm = o.GetName()
            k = self.__keys.FindObject(nm)
            o.Write()
            if k:
                k = self.__file.GetKey(nm)
                pid = "%s;%d" % (nm, k.GetCycle())
            else:
                pid = nm + ";1"
            return pid

class ROOT_Proxy:
    def __init__(self, f, pid):
        self.__f = f
        self.__pid = pid
        self.__o = None

    def __getattr__(self, a):
        if self.__o == None:
            self.__o = self.__f.Get(self.__pid)
            if self.__o.__class__.__module__ != 'ROOT':
                self.__o.__class__.__module__ = 'ROOT'
        return getattr(self.__o, a)

    def __obj(self):
        if self.__o == None:
            self.__o = self.__f.Get(self.__pid)
            if self.__o.__class__.__module__ != 'ROOT':
                self.__o.__class__.__module__ = 'ROOT'
        return self.__o

class Unpickler:
    def __init__(self, file, use_proxy=True, use_hash=False):
        """Create a ROOT unpickler.
        `file` should be a ROOT TFile.
        """
        global xserial
        xserial += 1
        self.__use_proxy = use_proxy
        self.__file = file
        self.__io = IO_Wrapper()
        self.__unpickle = cPickle.Unpickler (self.__io)
        self.__unpickle.persistent_load = self._persistent_load
        self.__unpickle.find_global = self._find_class
        self.__n = 0
        self.__serial = '%d-' % xserial
        xdict[self.__serial] = file

        if use_hash:
            htab = {}
            ctab = {}
            for k in file.GetListOfKeys():
                nm = k.GetName()
                cy = k.GetCycle()
                htab[(nm,cy)] = k
                if cy > ctab.get(nm, 0):
                    ctab[nm] = cy
                    htab[(nm, 9999)] = k
            file._htab = htab
            oget = file.Get
            def xget (nm0):
                nm = nm0
                ipos = nm.find(';')
                if ipos >= 0:
                    cy = nm[ipos+1]
                    if cy == '*':
                        cy = 10000
                    else:
                        cy = int(cy)
                    nm = nm[:ipos-1]
                else:
                    cy = 9999
                ret = htab.get((nm, cy), None)
                if not ret:
                    log.warning(
                        "did't find {0} {1} {2}".format(nm, cy, len(htab)))
                    return oget (nm0)
                #ctx = ROOT.TDirectory.TContext (file)
                ret = ret.ReadObj()
                #del ctx
                return ret
            file.Get = xget

    def load(self, key=None):
        """Read a pickled object representation from the open file."""
        if key == None:
            key = '_pickle'
        o = None
        if _compat_hooks:
            save = _compat_hooks[0]()
        try:
            self.__n += 1
            s = self.__file.Get(key + ';%d' % self.__n)
            self.__io.setvalue(s.GetName())
            o = self.__unpickle.load()
            self.__io.reopen()
        finally:
            if _compat_hooks:
                save = _compat_hooks[1](save)
        return o

    def _persistent_load(self, pid):
        if self.__use_proxy:
            o = ROOT_Proxy(self.__file, pid)
        else:
            o = self.__file.Get(pid)
        log.debug("load {0} {1}".format(pid, o))
        xdict[self.__serial + pid] = o
        return o

    def _find_class(self, module, name):
        try:
            try:
                __import__(module)
                mod = sys.modules[module]
            except ImportError:
                log.info("Making dummy module {0}".format(module))
                class DummyModule:
                    pass
                mod = DummyModule()
                sys.modules[module] = mod
            klass = getattr(mod, name)
            return klass
        except AttributeError:
            log.info("Making dummy class {0}.{1}".format(module, name))
            mod = sys.modules[module]
            class Dummy(object):
                pass
            setattr (mod, name, Dummy)
            return Dummy

def compat_hooks(hooks):
    """Set compatibility hooks.
    If this is set, then hooks[0] is called before loading, and hooks[1] is
    called after loading.  hooks[1] is called with the return value of hooks[0]
    as an argument.  This is useful for backwards compatibility in some
    situations.
    """
    global _compat_hooks
    _compat_hooks = hooks

def dump(o, f, proto=0, key=None):
    """Dump object O to the ROOT TFile `f`.

    `f` may be an open ROOT file or directory, or a string path to an existing
    ROOT file.
    """
    if isinstance(f, basestring):
        f = root_open(f, 'recreate')
        own_file = True
    else:
        own_file = False
    ret = Pickler(f, proto).dump(o, key)
    if own_file:
        f.Close()
    return ret

def load(f, use_proxy=1, key=None):
    """Load an object from the ROOT TFile `f`.

    `f` may be an open ROOT file or directory, or a string path to an existing
    ROOT file.
    """
    if isinstance(f, basestring):
        f = root_open(f)
    return Unpickler(f, use_proxy).load(key)
<commit_msg>docstring<commit_after># Copyright 2012 the rootpy developers
# distributed under the terms of the GNU General Public License
# Original author: Scott Snyder scott.snyder(a)cern.ch, 2004.

"""Pickle python data into a ROOT file, preserving references to ROOT objects.

This module allows pickling python objects into a ROOT file. The python
objects may contain references to named ROOT objects. If one has set up a
structure of python objects to hold ROOT histograms, this provides a convenient
way of saving and restoring your histograms. The pickled python data are
stored in an additional string object in the ROOT file; any ROOT objects are
stored as usual. (Thus, ROOT files written by the pickler can be read just
like any other ROOT file if you don't care about the python data.)

Here's an example of writing a pickle::

   from rootpy.plotting import Hist
   from rootpy.io.pickler import dump
   hlist = []
   for i in range(10):
       hlist.append(Hist(10, 0, 10))
   dump(hlist, 'test.root')

This writes a list of histograms to test.root. The histograms may be read back
like this::

   from rootpy.io.pickler import load
   hlist = load('test.root')

The following additional notes apply:

* Pickling may not always work correctly for the case of python objects
  deriving from ROOT objects. It will probably also not work for the case of
  ROOT objects which do not derive from TObject.

* When the pickled data are being read, if a class doesn't exist,
  a dummy class with no methods will be used instead. This is different
  from the standard pickle behavior (where it would be an error), but it
  simplifies usage in the common case where the class is being used to hold
  histograms, and its methods are entirely concerned with filling the
  histograms.

* When restoring a reference to a ROOT object, the default behavior
  is to not read the ROOT object itself, but instead to create a proxy. The
  ROOT object will then be read the first time the proxy is accessed. This can
  help significantly with time and memory usage if you're only accessing a
  small fraction of the ROOT objects, but it does mean that you need to keep
  the ROOT file open. Pass use_proxy=0 to disable this behavior.

"""
from . import log; log = log[__name__]
from . import root_open
from ..context import preserve_current_directory

from cStringIO import StringIO
import cPickle
import ROOT
import sys

__all__ = [
    'dump',
    'load',
    'compat_hooks',
]

_compat_hooks = None
xdict = {}
xserial = 0

# Argh!  We can't store NULs in TObjStrings.
# But pickle protocols > 0 are binary protocols, and will get corrupted
# if we truncate at a NUL.
# So, when we save the pickle data, make the mappings:
#  0x00 -> 0xff 0x01
#  0xff -> 0xff 0xfe
def _protect(s):
    return s.replace('\377', '\377\376').replace('\000', '\377\001')
def _restore(s):
    return s.replace('\377\001', '\000').replace('\377\376', '\377')

class IO_Wrapper:
    def __init__(self):
        return self.reopen()

    def write(self, s):
        return self.__s.write(_protect(s))

    def read(self, i):
        return self.__s.read(i)

    def readline(self):
        return self.__s.readline()

    def getvalue(self):
        return self.__s.getvalue()

    def setvalue(self, s):
        self.__s = StringIO(_restore(s))
        return

    def reopen(self):
        self.__s = StringIO()
        return

class Pickler:
    def __init__(self, file, proto=0):
        """Create a root pickler.
        `file` should be a ROOT TFile. `proto` is the python pickle protocol
        version to use.  The python part will be pickled to a ROOT
        TObjString called _pickle; it will contain references to the
        ROOT objects.
        """
        self.__file = file
        self.__keys = file.GetListOfKeys()
        self.__io = IO_Wrapper()
        self.__pickle = cPickle.Pickler(self.__io, proto)
        self.__pickle.persistent_id = self._persistent_id
        self.__pmap = {}

    def dump(self, o, key=None):
        """Write a pickled representation of o to the open TFile."""
        if key == None:
            key = '_pickle'
        with preserve_current_directory():
            self.__file.cd()
            self.__pickle.dump(o)
            s = ROOT.TObjString(self.__io.getvalue())
            self.__io.reopen()
            s.Write(key)
            self.__file.GetFile().Flush()
            self.__pmap.clear()

    def clear_memo(self):
        """Clears the pickler's internal memo."""
        self.__pickle.memo.clear()

    def _persistent_id(self, o):
        if hasattr(o, '_ROOT_Proxy__obj'):
            o = o._ROOT_Proxy__obj()
        if isinstance(o, ROOT.TObject):
            # Write the object, and return the resulting NAME;CYCLE.
            # We used to to this like this:
            #o.Write()
            #k = self.__file.GetKey (o.GetName())
            #pid = "%s;%d" % (k.GetName(), k.GetCycle())
            # It turns out, though, that destroying the python objects
            # referencing the TKeys is quite expensive (O(logN) where
            # N is the total number of pyroot objects?).  Although
            # we want to allow for the case of saving multiple objects
            # with the same name, the most common case is that the name
            # has not already been written to the file.  So we optimize
            # for that case, doing the key lookup before we write the
            # object, not after.  (Note further: GetKey() is very slow
            # if the key does not actually exist, as it does a linear
            # search of the key list.  We use FindObject instead for the
            # initial lookup, which is a hashed lookup, but it is not
            # guaranteed to find the highest cycle.  So if we do
            # find an existing key, we need to look up again using GetKey.
            nm = o.GetName()
            k = self.__keys.FindObject(nm)
            o.Write()
            if k:
                k = self.__file.GetKey(nm)
                pid = "%s;%d" % (nm, k.GetCycle())
            else:
                pid = nm + ";1"
            return pid

class ROOT_Proxy:
    def __init__(self, f, pid):
        self.__f = f
        self.__pid = pid
        self.__o = None

    def __getattr__(self, a):
        if self.__o == None:
            self.__o = self.__f.Get(self.__pid)
            if self.__o.__class__.__module__ != 'ROOT':
                self.__o.__class__.__module__ = 'ROOT'
        return getattr(self.__o, a)

    def __obj(self):
        if self.__o == None:
            self.__o = self.__f.Get(self.__pid)
            if self.__o.__class__.__module__ != 'ROOT':
                self.__o.__class__.__module__ = 'ROOT'
        return self.__o

class Unpickler:
    def __init__(self, file, use_proxy=True, use_hash=False):
        """Create a ROOT unpickler.
        `file` should be a ROOT TFile.
        """
        global xserial
        xserial += 1
        self.__use_proxy = use_proxy
        self.__file = file
        self.__io = IO_Wrapper()
        self.__unpickle = cPickle.Unpickler (self.__io)
        self.__unpickle.persistent_load = self._persistent_load
        self.__unpickle.find_global = self._find_class
        self.__n = 0
        self.__serial = '%d-' % xserial
        xdict[self.__serial] = file

        if use_hash:
            htab = {}
            ctab = {}
            for k in file.GetListOfKeys():
                nm = k.GetName()
                cy = k.GetCycle()
                htab[(nm,cy)] = k
                if cy > ctab.get(nm, 0):
                    ctab[nm] = cy
                    htab[(nm, 9999)] = k
            file._htab = htab
            oget = file.Get
            def xget (nm0):
                nm = nm0
                ipos = nm.find(';')
                if ipos >= 0:
                    cy = nm[ipos+1]
                    if cy == '*':
                        cy = 10000
                    else:
                        cy = int(cy)
                    nm = nm[:ipos-1]
                else:
                    cy = 9999
                ret = htab.get((nm, cy), None)
                if not ret:
                    log.warning(
                        "did't find {0} {1} {2}".format(nm, cy, len(htab)))
                    return oget (nm0)
                #ctx = ROOT.TDirectory.TContext (file)
                ret = ret.ReadObj()
                #del ctx
                return ret
            file.Get = xget

    def load(self, key=None):
        """Read a pickled object representation from the open file."""
        if key == None:
            key = '_pickle'
        o = None
        if _compat_hooks:
            save = _compat_hooks[0]()
        try:
            self.__n += 1
            s = self.__file.Get(key + ';%d' % self.__n)
            self.__io.setvalue(s.GetName())
            o = self.__unpickle.load()
            self.__io.reopen()
        finally:
            if _compat_hooks:
                save = _compat_hooks[1](save)
        return o

    def _persistent_load(self, pid):
        if self.__use_proxy:
            o = ROOT_Proxy(self.__file, pid)
        else:
            o = self.__file.Get(pid)
        log.debug("load {0} {1}".format(pid, o))
        xdict[self.__serial + pid] = o
        return o

    def _find_class(self, module, name):
        try:
            try:
                __import__(module)
                mod = sys.modules[module]
            except ImportError:
                log.info("Making dummy module {0}".format(module))
                class DummyModule:
                    pass
                mod = DummyModule()
                sys.modules[module] = mod
            klass = getattr(mod, name)
            return klass
        except AttributeError:
            log.info("Making dummy class {0}.{1}".format(module, name))
            mod = sys.modules[module]
            class Dummy(object):
                pass
            setattr (mod, name, Dummy)
            return Dummy

def compat_hooks(hooks):
    """Set compatibility hooks.
    If this is set, then hooks[0] is called before loading, and hooks[1] is
    called after loading.  hooks[1] is called with the return value of hooks[0]
    as an argument.  This is useful for backwards compatibility in some
    situations.
    """
    global _compat_hooks
    _compat_hooks = hooks

def dump(o, f, proto=0, key=None):
    """Dump object O to the ROOT TFile `f`.

    `f` may be an open ROOT file or directory, or a string path to an existing
    ROOT file.
    """
    if isinstance(f, basestring):
        f = root_open(f, 'recreate')
        own_file = True
    else:
        own_file = False
    ret = Pickler(f, proto).dump(o, key)
    if own_file:
        f.Close()
    return ret

def load(f, use_proxy=1, key=None):
    """Load an object from the ROOT TFile `f`.

    `f` may be an open ROOT file or directory, or a string path to an existing
    ROOT file.
    """
    if isinstance(f, basestring):
        f = root_open(f)
    return Unpickler(f, use_proxy).load(key)
<|endoftext|>


<commit_before>"""Routes WSGI Middleware"""
import re
import urllib

try:
    from paste.wsgiwrappers import WSGIRequest
except:
    pass

from routes.base import request_config

class RoutesMiddleware(object):
    def __init__(self, wsgi_app, mapper, use_method_override=True,
                 path_info=True):
        """Create a Route middleware object

        Using the use_method_override keyword will require Paste to be
        installed, and your application should use Paste's WSGIRequest object
        as it will properly handle POST issues with wsgi.input should Routes
        check it.

        If path_info is True, then should a route var contain path_info, the
        SCRIPT_NAME and PATH_INFO will be altered accordingly. This should be
        used with routes like:

        .. code-block:: Python

            map.connect('blog/*path_info', controller='blog', path_info='')
        """
        self.app = wsgi_app
        self.mapper = mapper
        self.use_method_override = use_method_override
        self.path_info = path_info

    def __call__(self, environ, start_response):
        config = request_config()
        config.mapper = self.mapper

        old_method = None
        if self.use_method_override:
            req = WSGIRequest(environ)
            if '_method' in environ.get('QUERY_STRING', '') and '_method' in req.GET:
                old_method = environ['REQUEST_METHOD']
                environ['REQUEST_METHOD'] = req.GET['_method']
            elif environ['REQUEST_METHOD'] == 'POST' and '_method' in req.POST:
                old_method = environ['REQUEST_METHOD']
                environ['REQUEST_METHOD'] = req.POST['_method']

        config.environ = environ
        match = config.mapper_dict

        if old_method:
            environ['REQUEST_METHOD'] = old_method

        if not match:
            return self.not_found(environ, start_response)

        for k,v in match.iteritems():
            if v:
                match[k] = urllib.unquote_plus(v)

        environ['wsgiorg.routing_args'] = ((), match)

        # If the route included a path_info attribute and it should be used to
        # alter the environ, we'll pull it out
        if self.path_info and match.get('path_info'):
            oldpath = environ['PATH_INFO']
            newpath = match.get('path_info') or ''
            environ['PATH_INFO'] = newpath
            if not environ['PATH_INFO'].startswith('/'):
                environ['PATH_INFO'] = '/' + environ['PATH_INFO']
            environ['SCRIPT_NAME'] += re.sub(r'^(.*?)/' + newpath + '$', r'\1', oldpath)
            if environ['SCRIPT_NAME'].endswith('/'):
                environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'][:-1]

        return self.app(environ, start_response)

    def not_found(self, environ, start_response):
        start_response('404 Not Found', [('Content-type', 'text/plain')])
        return ['Not found']
<commit_msg>[svn] Fixed middleware returning 404 when it should just populate an empty routing_args field.<commit_after>"""Routes WSGI Middleware"""
import re
import urllib

try:
    from paste.wsgiwrappers import WSGIRequest
except:
    pass

from routes.base import request_config

class RoutesMiddleware(object):
    def __init__(self, wsgi_app, mapper, use_method_override=True,
                 path_info=True):
        """Create a Route middleware object

        Using the use_method_override keyword will require Paste to be
        installed, and your application should use Paste's WSGIRequest object
        as it will properly handle POST issues with wsgi.input should Routes
        check it.

        If path_info is True, then should a route var contain path_info, the
        SCRIPT_NAME and PATH_INFO will be altered accordingly. This should be
        used with routes like:

        .. code-block:: Python

            map.connect('blog/*path_info', controller='blog', path_info='')
        """
        self.app = wsgi_app
        self.mapper = mapper
        self.use_method_override = use_method_override
        self.path_info = path_info

    def __call__(self, environ, start_response):
        config = request_config()
        config.mapper = self.mapper

        old_method = None
        if self.use_method_override:
            req = WSGIRequest(environ)
            if '_method' in environ.get('QUERY_STRING', '') and '_method' in req.GET:
                old_method = environ['REQUEST_METHOD']
                environ['REQUEST_METHOD'] = req.GET['_method']
            elif environ['REQUEST_METHOD'] == 'POST' and '_method' in req.POST:
                old_method = environ['REQUEST_METHOD']
                environ['REQUEST_METHOD'] = req.POST['_method']

        config.environ = environ
        match = config.mapper_dict

        if old_method:
            environ['REQUEST_METHOD'] = old_method

        if not match:
            match = {}

        for k,v in match.iteritems():
            if v:
                match[k] = urllib.unquote_plus(v)

        environ['wsgiorg.routing_args'] = ((), match)

        # If the route included a path_info attribute and it should be used to
        # alter the environ, we'll pull it out
        if self.path_info and match.get('path_info'):
            oldpath = environ['PATH_INFO']
            newpath = match.get('path_info') or ''
            environ['PATH_INFO'] = newpath
            if not environ['PATH_INFO'].startswith('/'):
                environ['PATH_INFO'] = '/' + environ['PATH_INFO']
            environ['SCRIPT_NAME'] += re.sub(r'^(.*?)/' + newpath + '$', r'\1', oldpath)
            if environ['SCRIPT_NAME'].endswith('/'):
                environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'][:-1]

        return self.app(environ, start_response)
    <|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import time
import logging
import json
from math import floor

from cloud4rpi import config
from cloud4rpi import utils
from cloud4rpi.errors import MqttConnectionError

import paho.mqtt.client as mqtt

KEEP_ALIVE_INTERVAL = 30
MQTT_ERR_SUCCESS = 0
CONNECT_RESULT_UNDEFINED = 255

log = logging.getLogger(config.loggerName)

class MqttApi(object):
    def __init__(self,
                 device_token,
                 host=config.mqqtBrokerHost,
                 port=config.mqttBrokerPort):
        utils.guard_against_invalid_token(device_token)

        def noop_on_command(cmd):
            pass

        self.__device_token = device_token
        self.__client = mqtt.Client(device_token, clean_session=False)
        self.__host = host
        self.__port = port
        self.__connect_result = None

        self.on_command = noop_on_command

    def __format_topic(self, tail):
        return 'devices/{0}/{1}'.format(self.__device_token, tail)

    def connect(self):
        def on_connect(client, userdata, flags, rc):
            self.__connect_result = rc
            if rc != MQTT_ERR_SUCCESS:
                log.error('Connection failed: %s', rc)
                raise MqttConnectionError(rc)
            topic = self.__format_topic('commands')
            log.debug('Listen for %s', topic)
            self.__client.subscribe(topic, qos=1)

        def on_message(client, userdata, msg):
            log.info('Command received %s: %s', msg.topic, msg.payload)
            if callable(self.on_command):
                payload = msg.payload.decode() \
                    if isinstance(msg.payload, bytes) else msg.payload
                self.on_command(json.loads(payload))

        def on_disconnect(client, userdata, rc):
            log.info('MQTT disconnected with code: %s', rc)
            self.__on_disconnect(rc)

        self.__client.on_connect = on_connect
        self.__client.on_message = on_message
        self.__client.on_disconnect = on_disconnect

        log.info('MQTT connecting %s:%s', self.__host, self.__port)
        self.__client.connect(self.__host, self.__port,
                              keepalive=KEEP_ALIVE_INTERVAL)

        self.__connect_result = CONNECT_RESULT_UNDEFINED
        self.__client.loop_start()

        while self.__connect_result == CONNECT_RESULT_UNDEFINED:
            time.sleep(.01)

    def __on_disconnect(self, rc):
        if rc == MQTT_ERR_SUCCESS:
            return

        attempts = 0
        retry_interval = 0.5
        while True:
            attempts += 1
            log.info('Attempt #%s to reconnect after %s sec', attempts,
                     floor(retry_interval))
            retry_interval = min(retry_interval * 2, 100)
            try:
                self.__client.reconnect()
                log.info("Reconnected!")
                return
            except Exception as e:
                log.info('Reconnection failed: %s', str(e))
                time.sleep(retry_interval)

    def disconnect(self):
        self.__client.loop_stop()
        self.__client.disconnect()

    def publish_config(self, cfg):
        self.__publish(self.__format_topic('config'), cfg)

    def publish_data(self, data):
        self.__publish(self.__format_topic('data'), data)

    def publish_diag(self, diag):
        self.__publish(self.__format_topic('diagnostics'), diag)

    def __publish(self, topic, payload=None):
        if payload is None:
            return
        msg = {
            'ts': utils.utcnow(),
            'payload': payload,
        }
        log.info('Publishing %s: %s', topic, msg)
        self.__client.publish(topic, payload=json.dumps(msg))
<commit_msg>Publish messages with QoS 1 & improve logging (#16)<commit_after># -*- coding: utf-8 -*-

import time
import logging
import json

from cloud4rpi import config
from cloud4rpi import utils
from cloud4rpi.errors import MqttConnectionError

import paho.mqtt.client as mqtt

KEEP_ALIVE_INTERVAL = 30  # sec
RETRY_INTERVAL = 5  # sec
CONNECT_RESULT_UNDEFINED = 255

log = logging.getLogger(config.loggerName)

def is_success(rc):
    return rc == mqtt.MQTT_ERR_SUCCESS

class MqttApi(object):
    def __init__(self,
                 device_token,
                 host=config.mqqtBrokerHost,
                 port=config.mqttBrokerPort):
        utils.guard_against_invalid_token(device_token)

        def noop_on_command(cmd):
            pass

        self.__device_token = device_token
        self.__client = mqtt.Client(device_token, clean_session=False)
        self.__host = host
        self.__port = port
        self.__qos = 1
        self.__connect_result = None

        self.on_command = noop_on_command

    @property
    def commands_topic(self):
        return self.__format_topic('commands')

    @property
    def config_topic(self):
        return self.__format_topic('config')

    @property
    def data_topic(self):
        return self.__format_topic('data')

    @property
    def diag_topic(self):
        return self.__format_topic('diagnostics')

    def __format_topic(self, tail):
        return 'devices/{0}/{1}'.format(self.__device_token, tail)

    def connect(self):
        def on_connect(client, userdata, flags, rc):
            if not is_success(rc):
                log.error('Connection failed: %s', rc)
                raise MqttConnectionError(rc)

            log.info('Connected')
            self.__connect_result = rc

            log.info('Subscribing %s with QoS %s',
                     self.commands_topic, str(self.__qos))
            self.__client.subscribe(self.commands_topic, qos=self.__qos)

        def on_message(client, userdata, msg):
            log.info('Command received %s: %s', msg.topic, msg.payload)
            if callable(self.on_command):
                payload = msg.payload.decode() \
                    if isinstance(msg.payload, bytes) else msg.payload
                self.on_command(json.loads(payload))

        def on_disconnect(client, userdata, rc):
            log.info('Disconnected with code: %s', rc)
            self.__on_disconnect(rc)

        def on_publish(client, packet, mid):
            log.info('Published %s: %s', packet['topic'], packet['msg'])

        self.__client.on_connect = on_connect
        self.__client.on_message = on_message
        self.__client.on_disconnect = on_disconnect
        self.__client.on_publish = on_publish

        log.info('Connecting %s:%s', self.__host, self.__port)
        self.__client.connect(self.__host, self.__port,
                              keepalive=KEEP_ALIVE_INTERVAL)

        self.__connect_result = CONNECT_RESULT_UNDEFINED
        self.__client.loop_start()

        while self.__connect_result == CONNECT_RESULT_UNDEFINED:
            time.sleep(.01)

    def __on_disconnect(self, rc):
        if is_success(rc):
            return

        while True:
            log.info('Reconnecting')

            try:
                self.__client.reconnect()
                return
            except Exception as e:
                log.info('Reconnection failed: %s', str(e))

            time.sleep(RETRY_INTERVAL)

    def disconnect(self):
        self.__client.loop_stop()
        self.__client.disconnect()

    def publish_config(self, msg):
        self.__publish(self.config_topic, msg)

    def publish_data(self, msg):
        self.__publish(self.data_topic, msg)

    def publish_diag(self, msg):
        self.__publish(self.diag_topic, msg)

    def __publish(self, topic, payload=None):
        if payload is None:
            return

        msg = {
            'ts': utils.utcnow(),
            'payload': payload,
        }

        self.__client.user_data_set({
            'topic': topic,
            'msg': msg,
        })

        self.__client.publish(topic, qos=self.__qos, payload=json.dumps(msg))
<|endoftext|>


<commit_before>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

REST_HOST_KEY = 'REST_HOST'
REST_PORT_KEY = 'REST_PORT'
CELERY_WORK_DIR_KEY = 'CELERY_WORK_DIR'
MANAGER_FILE_SERVER_URL_KEY = 'MANAGER_FILE_SERVER_URL'
MANAGER_FILE_SERVER_ROOT_KEY = 'MANAGER_FILE_SERVER_ROOT'
FILE_SERVER_RESOURCES_FOLDER = 'resources'
FILE_SERVER_BLUEPRINTS_FOLDER = 'blueprints'
FILE_SERVER_DEPLOYMENTS_FOLDER = 'deployments'
FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER = 'uploaded-blueprints'
FILE_SERVER_SNAPSHOTS_FOLDER = 'snapshots'
FILE_SERVER_PLUGINS_FOLDER = 'plugins'

AGENT_INSTALL_METHOD_NONE = 'none'
AGENT_INSTALL_METHOD_REMOTE = 'remote'
AGENT_INSTALL_METHOD_INIT_SCRIPT = 'init_script'
AGENT_INSTALL_METHOD_PROVIDED = 'provided'
AGENT_INSTALL_METHOD_PLUGIN = 'plugin'
AGENT_INSTALL_METHODS = [
    AGENT_INSTALL_METHOD_NONE,
    AGENT_INSTALL_METHOD_REMOTE,
    AGENT_INSTALL_METHOD_INIT_SCRIPT,
    AGENT_INSTALL_METHOD_PROVIDED,
    AGENT_INSTALL_METHOD_PLUGIN
]
AGENT_INSTALL_METHODS_SCRIPTS = [
    AGENT_INSTALL_METHOD_INIT_SCRIPT,
    AGENT_INSTALL_METHOD_PROVIDED,
    AGENT_INSTALL_METHOD_PLUGIN
]

COMPUTE_NODE_TYPE = 'cloudify.nodes.Compute'

BROKER_PORT_NO_SSL = 5672
BROKER_PORT_SSL = 5671
CELERY_TASK_RESULT_EXPIRES = 600
CLOUDIFY_TOKEN_AUTHENTICATION_HEADER = 'Authentication-Token'
LOCAL_REST_CERT_FILE_KEY = 'LOCAL_REST_CERT_FILE'
SECURED_PROTOCOL = 'https'

BROKER_SSL_CERT_PATH = 'BROKER_SSL_CERT_PATH'
BYPASS_MAINTENANCE = 'BYPASS_MAINTENANCE'
LOGGING_CONFIG_FILE = '/etc/cloudify/logging.conf'
CLUSTER_SETTINGS_PATH_KEY = 'CLOUDIFY_CLUSTER_SETTINGS_PATH'

MGMTWORKER_QUEUE = 'cloudify.management'
DEPLOYMENT = 'deployment'
NODE_INSTANCE = 'node-instance'
RELATIONSHIP_INSTANCE = 'relationship-instance'

DEFAULT_NETWORK_NAME = 'default'
<commit_msg>CY-289-custom-resource-dir (#401)<commit_after>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

REST_HOST_KEY = 'REST_HOST'
REST_PORT_KEY = 'REST_PORT'
CELERY_WORK_DIR_KEY = 'CELERY_WORK_DIR'
MANAGER_FILE_SERVER_URL_KEY = 'MANAGER_FILE_SERVER_URL'
MANAGER_FILE_SERVER_ROOT_KEY = 'MANAGER_FILE_SERVER_ROOT'
FILE_SERVER_RESOURCES_FOLDER = 'resources'
FILE_SERVER_BLUEPRINTS_FOLDER = 'blueprints'
FILE_SERVER_DEPLOYMENTS_FOLDER = 'deployments'
FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER = 'uploaded-blueprints'
FILE_SERVER_SNAPSHOTS_FOLDER = 'snapshots'
FILE_SERVER_PLUGINS_FOLDER = 'plugins'
FILE_SERVER_GLOBAL_RESOURCES_FOLDER = 'global-resources'
FILE_SERVER_TENANT_RESOURCES_FOLDER = 'tenant-resources'

AGENT_INSTALL_METHOD_NONE = 'none'
AGENT_INSTALL_METHOD_REMOTE = 'remote'
AGENT_INSTALL_METHOD_INIT_SCRIPT = 'init_script'
AGENT_INSTALL_METHOD_PROVIDED = 'provided'
AGENT_INSTALL_METHOD_PLUGIN = 'plugin'
AGENT_INSTALL_METHODS = [
    AGENT_INSTALL_METHOD_NONE,
    AGENT_INSTALL_METHOD_REMOTE,
    AGENT_INSTALL_METHOD_INIT_SCRIPT,
    AGENT_INSTALL_METHOD_PROVIDED,
    AGENT_INSTALL_METHOD_PLUGIN
]
AGENT_INSTALL_METHODS_SCRIPTS = [
    AGENT_INSTALL_METHOD_INIT_SCRIPT,
    AGENT_INSTALL_METHOD_PROVIDED,
    AGENT_INSTALL_METHOD_PLUGIN
]

COMPUTE_NODE_TYPE = 'cloudify.nodes.Compute'

BROKER_PORT_NO_SSL = 5672
BROKER_PORT_SSL = 5671
CELERY_TASK_RESULT_EXPIRES = 600
CLOUDIFY_TOKEN_AUTHENTICATION_HEADER = 'Authentication-Token'
LOCAL_REST_CERT_FILE_KEY = 'LOCAL_REST_CERT_FILE'
SECURED_PROTOCOL = 'https'

BROKER_SSL_CERT_PATH = 'BROKER_SSL_CERT_PATH'
BYPASS_MAINTENANCE = 'BYPASS_MAINTENANCE'
LOGGING_CONFIG_FILE = '/etc/cloudify/logging.conf'
CLUSTER_SETTINGS_PATH_KEY = 'CLOUDIFY_CLUSTER_SETTINGS_PATH'

MGMTWORKER_QUEUE = 'cloudify.management'
DEPLOYMENT = 'deployment'
NODE_INSTANCE = 'node-instance'
RELATIONSHIP_INSTANCE = 'relationship-instance'

DEFAULT_NETWORK_NAME = 'default'
<|endoftext|>


<commit_before>from datetime import datetime
import argparse
import os

def is_dir(string):
    """
    Type check for a valid directory for ArgumentParser.
    """
    if not os.path.isdir(string):
        msg = '{0} is not a directory'.format(string)
        raise argparse.ArgumentTypeError(msg)
    return string

def is_file(string):
    """
    Type check for a valid file for ArgumentParser.
    """
    if not os.path.isfile(string):
        msg = u'{0} is not a file'.format(string)
        raise argparse.ArgumentTypeError(msg)
    return string

def gt_zero(string):
    """
    Type check for int > 0 for ArgumentParser.
    """
    if not int(string) > 0:
        msg = u'limit must be > 0'
        raise argparse.ArgumentTypeError(msg)
    return int(string)

def isodate(string):
    try:
        return datetime.strptime(string, '%Y-%m-%d').date()
    except ValueError:
        msg = u'date input must in the format of yyyy-mm-dd'
        raise argparse.ArgumentTypeError(msg)
<commit_msg>Fix bug with self reference<commit_after>from __future__ import absolute_import

from datetime import datetime
import argparse as _argparse
import os

def is_dir(string):
    """
    Type check for a valid directory for ArgumentParser.
    """
    if not os.path.isdir(string):
        msg = '{0} is not a directory'.format(string)
        raise _argparse.ArgumentTypeError(msg)
    return string

def is_file(string):
    """
    Type check for a valid file for ArgumentParser.
    """
    if not os.path.isfile(string):
        msg = u'{0} is not a file'.format(string)
        raise _argparse.ArgumentTypeError(msg)
    return string

def gt_zero(string):
    """
    Type check for int > 0 for ArgumentParser.
    """
    if not int(string) > 0:
        msg = u'limit must be > 0'
        raise _argparse.ArgumentTypeError(msg)
    return int(string)

def isodate(string):
    try:
        return datetime.strptime(string, '%Y-%m-%d').date()
    except ValueError:
        msg = u'date input must in the format of yyyy-mm-dd'
        raise _argparse.ArgumentTypeError(msg)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from cms.api import create_page
from cms.models import Page
from cms.test_utils.util.context_managers import SettingsOverride
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.db import connections
from django.utils import unittest
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, NoAlertPresentException
from django.test import LiveServerTestCase, TestCase
import os

class CMSLiveTests(LiveServerTestCase, TestCase):
    @classmethod
    def setUpClass(cls):
        super(CMSLiveTests, cls).setUpClass()
        if os.environ.get('SELENIUM', '') != '':
            #skip selenium tests
            raise unittest.SkipTest("Selenium env is set to 0")
        if os.environ.get("TRAVIS_BUILD_NUMBER"):
            capabilities = webdriver.DesiredCapabilities.CHROME
            capabilities['version'] = '31'
            capabilities['platform'] = 'OS X 10.9'
            capabilities['name'] = 'django CMS'
            capabilities['build'] = os.environ.get("TRAVIS_BUILD_NUMBER")
            capabilities['tags'] = [os.environ.get("TRAVIS_PYTHON_VERSION"), "CI"]
            username = os.environ.get("SAUCE_USERNAME")
            access_key = os.environ.get("SAUCE_ACCESS_KEY")
            capabilities["tunnel-identifier"] = os.environ.get("TRAVIS_JOB_NUMBER")
            hub_url = "http://%s:%s@ondemand.saucelabs.com/wd/hub" % (username, access_key)
            cls.driver = webdriver.Remote(desired_capabilities=capabilities, command_executor=hub_url)
            cls.driver.implicitly_wait(30)
        else:
            cls.driver = webdriver.Firefox()
            cls.driver.implicitly_wait(5)
        cls.accept_next_alert = True

    @classmethod
    def tearDownClass(cls):
        cls.driver.quit()
        super(CMSLiveTests, cls).tearDownClass()

    def tearDown(self):
        super(CMSLiveTests, self).tearDown()
        Page.objects.all().delete() # somehow the sqlite transaction got lost.

    def wait_until(self, callback, timeout=10):
        """
        Helper function that blocks the execution of the tests until the
        specified callback returns a value that is not falsy. This function can
        be called, for example, after clicking a link or submitting a form.
        See the other public methods that call this function for more details.
        """
        from selenium.webdriver.support.wait import WebDriverWait

        WebDriverWait(self.driver, timeout).until(callback)

    def wait_loaded_tag(self, tag_name, timeout=10):
        """
        Helper function that blocks until the element with the given tag name
        is found on the page.
        """
        self.wait_until(
            lambda driver: driver.find_element_by_tag_name(tag_name),
            timeout
        )

    def wait_page_loaded(self):
        """
        Block until page has started to load.
        """
        from selenium.common.exceptions import TimeoutException

        try:
            # Wait for the next page to be loaded
            self.wait_loaded_tag('body')
        except TimeoutException:
            # IE7 occasionnally returns an error "Internet Explorer cannot
            # display the webpage" and doesn't load the next page. We just
            # ignore it.
            pass

    def is_element_present(self, how, what):
        try:
            self.driver.find_element(by=how, value=what)
        except NoSuchElementException:
            return False
        return True

    def is_alert_present(self):
        try:
            self.driver.switch_to_alert()
        except NoAlertPresentException:
            return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally:
            self.accept_next_alert = True

class ToolbarBasicTests(CMSLiveTests):

    def setUp(self):
        Site.objects.create(domain='example.org', name='example.org')
        self.base_url = self.live_server_url
        user = User()
        user.username = 'admin'
        user.set_password('admin')
        user.is_superuser = user.is_staff = user.is_active = True
        user.save()
        self.driver.implicitly_wait(2)
        super(ToolbarBasicTests, self).setUp()

    def test_toolbar_login(self):
        create_page('Home', 'simple.html', 'en', published=True)
        url = '%s/?edit' % self.live_server_url
        self.assertTrue(User.objects.all().count(), 1)
        self.driver.get(url)
        self.assertRaises(NoSuchElementException, self.driver.find_element_by_class_name, 'cms_toolbar-item_logout')
        username_input = self.driver.find_element_by_id("id_cms-username")
        username_input.send_keys('admin')
        password_input = self.driver.find_element_by_id("id_cms-password")
        password_input.send_keys('admin')
        password_input.submit()
        self.wait_page_loaded()
        self.assertTrue(self.driver.find_element_by_class_name('cms_toolbar-item-navigation'))

    def test_basic_add_pages(self):
        with SettingsOverride(DEBUG=True):
            self.assertEqual(Page.objects.all().count(), 0)
            self.assertTrue(User.objects.all().count(), 1)
            driver = self.driver
            driver.get(self.base_url + "/de/")
            driver.find_element_by_id("add-page").click()
            driver.find_element_by_id("id_username").clear()
            driver.find_element_by_id("id_username").send_keys("admin")
            driver.find_element_by_id("id_password").clear()
            driver.find_element_by_id("id_password").send_keys("admin")
            driver.find_element_by_css_selector("input[type=\"submit\"]").click()
            driver.find_element_by_name("_save").click()
            driver.find_element_by_link_text(u"Seite hinzufügen").click()
            driver.find_element_by_id("id_title").clear()
            driver.find_element_by_id("id_title").send_keys("SubPage")
            driver.find_element_by_name("_save").click()

<commit_msg>Added 2 frontend tests for checking if the clipboard copy and the copy from language item work.<commit_after># -*- coding: utf-8 -*-
import os
import time

from cms.models import Page
from cms.api import create_page, create_title, add_plugin
from cms.test_utils.testcases import SettingsOverrideTestCase
from cms.test_utils.util.context_managers import SettingsOverride

from django.utils import unittest
from django.db import connections
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.test import LiveServerTestCase, TestCase

from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, NoAlertPresentException

class CMSLiveTests(LiveServerTestCase, TestCase):
    @classmethod
    def setUpClass(cls):
        super(CMSLiveTests, cls).setUpClass()
        if os.environ.get('SELENIUM', '') != '':
            #skip selenium tests
            raise unittest.SkipTest("Selenium env is set to 0")
        if os.environ.get("TRAVIS_BUILD_NUMBER"):
            capabilities = webdriver.DesiredCapabilities.CHROME
            capabilities['version'] = '31'
            capabilities['platform'] = 'OS X 10.9'
            capabilities['name'] = 'django CMS'
            capabilities['build'] = os.environ.get("TRAVIS_BUILD_NUMBER")
            capabilities['tags'] = [os.environ.get("TRAVIS_PYTHON_VERSION"), "CI"]
            username = os.environ.get("SAUCE_USERNAME")
            access_key = os.environ.get("SAUCE_ACCESS_KEY")
            capabilities["tunnel-identifier"] = os.environ.get("TRAVIS_JOB_NUMBER")
            hub_url = "http://%s:%s@ondemand.saucelabs.com/wd/hub" % (username, access_key)
            cls.driver = webdriver.Remote(desired_capabilities=capabilities, command_executor=hub_url)
            cls.driver.implicitly_wait(30)
        else:
            cls.driver = webdriver.Firefox()
            cls.driver.implicitly_wait(5)
        cls.accept_next_alert = True

    @classmethod
    def tearDownClass(cls):
        cls.driver.quit()
        super(CMSLiveTests, cls).tearDownClass()

    def tearDown(self):
        super(CMSLiveTests, self).tearDown()
        Page.objects.all().delete() # somehow the sqlite transaction got lost.

    def wait_until(self, callback, timeout=10):
        """
        Helper function that blocks the execution of the tests until the
        specified callback returns a value that is not falsy. This function can
        be called, for example, after clicking a link or submitting a form.
        See the other public methods that call this function for more details.
        """
        from selenium.webdriver.support.wait import WebDriverWait

        WebDriverWait(self.driver, timeout).until(callback)

    def wait_loaded_tag(self, tag_name, timeout=10):
        """
        Helper function that blocks until the element with the given tag name
        is found on the page.
        """
        self.wait_until(
            lambda driver: driver.find_element_by_tag_name(tag_name),
            timeout
        )

    def wait_page_loaded(self):
        """
        Block until page has started to load.
        """
        from selenium.common.exceptions import TimeoutException

        try:
            # Wait for the next page to be loaded
            self.wait_loaded_tag('body')
        except TimeoutException:
            # IE7 occasionnally returns an error "Internet Explorer cannot
            # display the webpage" and doesn't load the next page. We just
            # ignore it.
            pass

    def is_element_present(self, how, what):
        try:
            self.driver.find_element(by=how, value=what)
        except NoSuchElementException:
            return False
        return True

    def is_alert_present(self):
        try:
            self.driver.switch_to_alert()
        except NoAlertPresentException:
            return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally:
            self.accept_next_alert = True

class ToolbarBasicTests(CMSLiveTests):

    def setUp(self):
        Site.objects.create(domain='example.org', name='example.org')
        self.base_url = self.live_server_url
        user = User()
        user.username = 'admin'
        user.set_password('admin')
        user.is_superuser = user.is_staff = user.is_active = True
        user.save()
        self.driver.implicitly_wait(2)
        super(ToolbarBasicTests, self).setUp()

    def test_toolbar_login(self):
        create_page('Home', 'simple.html', 'en', published=True)
        url = '%s/?edit' % self.live_server_url
        self.assertTrue(User.objects.all().count(), 1)
        self.driver.get(url)
        self.assertRaises(NoSuchElementException, self.driver.find_element_by_class_name, 'cms_toolbar-item_logout')
        username_input = self.driver.find_element_by_id("id_cms-username")
        username_input.send_keys('admin')
        password_input = self.driver.find_element_by_id("id_cms-password")
        password_input.send_keys('admin')
        password_input.submit()
        self.wait_page_loaded()
        self.assertTrue(self.driver.find_element_by_class_name('cms_toolbar-item-navigation'))

    def test_basic_add_pages(self):
        with SettingsOverride(DEBUG=True):
            self.assertEqual(Page.objects.all().count(), 0)
            self.assertTrue(User.objects.all().count(), 1)
            driver = self.driver
            driver.get(self.base_url + "/de/")
            driver.find_element_by_id("add-page").click()
            driver.find_element_by_id("id_username").clear()
            driver.find_element_by_id("id_username").send_keys("admin")
            driver.find_element_by_id("id_password").clear()
            driver.find_element_by_id("id_password").send_keys("admin")
            driver.find_element_by_css_selector("input[type=\"submit\"]").click()
            driver.find_element_by_name("_save").click()
            driver.find_element_by_link_text(u"Seite hinzufügen").click()
            driver.find_element_by_id("id_title").clear()
            driver.find_element_by_id("id_title").send_keys("SubPage")
            driver.find_element_by_name("_save").click()

class PlaceholderBasicTests(CMSLiveTests, SettingsOverrideTestCase):
    settings_overrides = {
        'LANGUAGE_CODE': 'en',
        'LANGUAGES': (('en', 'English'),
                      ('it', 'Italian')),
        'CMS_LANGUAGES': {
            1: [ {'code' : 'en',
                  'name': 'English',
                  'public': True},
                 {'code': 'it',
                  'name': 'Italian',
                  'public': True},
            ],
            'default': {
                'public': True,
                'hide_untranslated': False,
            }
        },
        'SITE_ID': 1,
    }

    def setUp(self):
        Site.objects.create(domain='example.org', name='example.org')

        self.page = create_page('Home', 'simple.html', 'en', published=True)
        self.italian_title = create_title('it', 'Home italian', self.page)

        self.placeholder = self.page.placeholders.all()[0]

        plugin = add_plugin(self.placeholder, 'TextPlugin', 'en', body='test')

        self.base_url = self.live_server_url

        user = User()
        user.username = 'admin'
        user.set_password('admin')
        user.is_superuser = user.is_staff = user.is_active = True
        user.save()

        self.driver.implicitly_wait(5)

        super(PlaceholderBasicTests, self).setUp()

    def _login(self):
        url = '%s/?edit' % self.live_server_url
        self.driver.get(url)

        self.assertRaises(NoSuchElementException, self.driver.find_element_by_class_name, 'cms_toolbar-item_logout')
        username_input = self.driver.find_element_by_id("id_cms-username")
        username_input.send_keys('admin')
        password_input = self.driver.find_element_by_id("id_cms-password")
        password_input.send_keys('admin')
        password_input.submit()
        self.wait_page_loaded()

        self.assertTrue(self.driver.find_element_by_class_name('cms_toolbar-item-navigation'))

    def test_copy_from_language(self):
        self._login()
        self.driver.get('%s/it/?edit' % self.live_server_url)

        build_button = self.driver.find_element_by_css_selector('.cms_toolbar-item-cms-mode-switcher a[href="?build"]')
        build_button.click()

        submenu = self.driver.find_element_by_css_selector('.cms_dragbar .cms_submenu')

        hov = ActionChains(self.driver).move_to_element(submenu)
        hov.perform()

        copy_from_english = self.driver.find_element_by_css_selector('.cms_submenu-item a[data-rel="copy-lang"][data-language="en"]')
        copy_from_english.click()

        # Done, check if the text plugin was copied

        time.sleep(1)

        italian_plugins = self.page.placeholders.all()[0].get_plugins_list('it')
        plugin_instance = italian_plugins[0].get_plugin_instance()[0]

        self.assertEqual(plugin_instance.body, 'test')

    def test_copy_to_from_clipboard(self):
        self._login()

        build_button = self.driver.find_element_by_css_selector('.cms_toolbar-item-cms-mode-switcher a[href="?build"]')
        build_button.click()

        cms_draggable = self.driver.find_element_by_css_selector('.cms_draggable:nth-child(1)')

        hov = ActionChains(self.driver).move_to_element(cms_draggable)
        hov.perform()

        submenu = cms_draggable.find_element_by_css_selector('.cms_submenu')

        hov = ActionChains(self.driver).move_to_element(submenu)
        hov.perform()

        copy = submenu.find_element_by_css_selector('a[data-rel="copy"]')
        copy.click()

        clipboard = self.driver.find_element_by_css_selector('.cms_clipboard')

        WebDriverWait(self.driver, 10).until(lambda driver: clipboard.is_displayed())

        hov = ActionChains(self.driver).move_to_element(clipboard)
        hov.perform()

        # necessary sleeps for making a "real" drag and drop, that works with the clipboard

        time.sleep(0.1)

        drag = ActionChains(self.driver).click_and_hold(
            clipboard.find_element_by_css_selector('.cms_draggable:nth-child(1)')
        );

        drag.perform()

        time.sleep(0.1)

        drag = ActionChains(self.driver).move_to_element(
            self.driver.find_element_by_css_selector('.cms_dragarea-1')
        )
        drag.perform()

        time.sleep(0.2)

        drag = ActionChains(self.driver).move_by_offset(
            0, 10
        ).release()

        drag.perform()

        plugins = self.page.placeholders.all()[0].get_plugins_list('en')

        self.assertEqual(len(plugins), 2)
<|endoftext|>