<commit_before># coding: utf-8

from __future__ import unicode_literals
import warnings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.generic import GenericForeignKey, \
                                                GenericRelation
from django.core.exceptions import ValidationError
from django.db import connection
from django.db.models import CharField, ForeignKey, ManyToManyField, \
    OneToOneField, BooleanField, PositiveSmallIntegerField, permalink, Q, \
    PositiveIntegerField, get_model, SmallIntegerField, PROTECT, Count
from django.utils.encoding import python_2_unicode_compatible, smart_text
from django.utils.html import strip_tags
from django.utils.translation import ungettext_lazy
from cache_tools import model_method_cached, cached_ugettext as ugettext, \
    cached_ugettext_lazy as _
from .common import CommonModel, AutoriteModel, LOWER_MSG, PLURAL_MSG, \
    calc_pluriel, CommonQuerySet, CommonManager, OrderedDefaultDict, \
    PublishedManager, PublishedQuerySet
from .functions import capfirst, str_list, str_list_w_last, href, hlp, \
    microdata

__all__ = (b'ElementDeDistribution', b'CaracteristiqueDElementDeProgramme',
           b'ElementDeProgramme', b'Evenement')

class ElementDeDistributionQuerySet(CommonQuerySet):
    def individus(self):
        return get_model('libretto', 'Individu').objects.filter(
            pk__in=self.values_list('individus', flat=True))

    def evenements(self):
        distributions_evenements = self.filter(
            content_type__app_label='libretto',
            content_type__model='evenement')
        pk_list = distributions_evenements.values_list('object_id', flat=True)
        return Evenement.objects.filter(Q(pk__in=pk_list)
                              | Q(programme__distribution__in=self)).distinct()

    def prefetch(self):
        return self.select_related(
            'pupitre', 'pupitre__partie',
            'profession').prefetch_related('individus')

    def html(self, tags=True):
        return ', '.join(e.html(tags=tags) for e in self)

class ElementDeDistributionManager(CommonManager):
    use_for_related_fields = True

    def get_query_set(self):
        return ElementDeDistributionQuerySet(self.model, using=self._db)

    def individus(self):
        return self.get_query_set().individus()

    def evenements(self):
        return self.get_query_set().evenements()

    def prefetch(self):
        return self.get_query_set().prefetch()

    def html(self, tags=True):
        return self.get_query_set().html(tags=tags)

@python_2_unicode_compatible
class ElementDeDistribution(CommonModel):
    individus = ManyToManyField('Individu', verbose_name=_('individus'),
                                related_name='elements_de_distribution')
    pupitre = ForeignKey(
        'Pupitre', verbose_name=_('pupitre'), null=True, blank=True,
        related_name='elements_de_distribution', on_delete=PROTECT)
    profession = ForeignKey(
        'Profession', verbose_name=_('profession'), null=True, blank=True,
        related_name='elements_de_distribution', on_delete=PROTECT)
    content_type = ForeignKey(ContentType, null=True, on_delete=PROTECT)
    object_id = PositiveIntegerField(null=True)
    content_object = GenericForeignKey()

    objects = ElementDeDistributionManager()

    class Meta(object):
        verbose_name = ungettext_lazy('élément de distribution',
                                      'éléments de distribution', 1)
        verbose_name_plural = ungettext_lazy('élément de distribution',
                                             'éléments de distribution', 2)
        ordering = ('pupitre',)
        app_label = 'libretto'

    def __str__(self):
        return self.html(tags=False)

    def html(self, tags=True):
        out = ''
        if self.pk:
            individus = self.individus.all()
            out += str_list_w_last(individu.html(tags=tags)
                                   for individu in individus)

        if self.pupitre:
            out += ' [' + self.pupitre.partie.link() + ']'
        elif self.profession:
            out += ' [' + self.profession.link() + ']'

        if not tags:
            return strip_tags(out)
        return out

    def related_label(self):
        return self.get_change_link()

    @permalink
    def get_change_url(self):
        return 'admin:libretto_elementdedistribution_change', (self.pk,)

    def get_change_link(self):
        return href(self.get_change_url(), smart_text(self), new_tab=True)

    def clean(self):
        if self.pupitre and self.profession:
            raise ValidationError(_('Vous ne pouvez remplir à la fois '
                                    '« Pupitre » et « Profession ».'))

    @staticmethod
    def autocomplete_search_fields():
        return (
            'pupitre__partie__nom__icontains',
            'individus__nom__icontains',
            'individus__pseudonyme__icontains',
        )

@python_2_unicode_compatible
class CaracteristiqueDElementDeProgramme(CommonModel):
    nom = CharField(_('nom'), max_length=100, help_text=LOWER_MSG, unique=True,
                    db_index=True)
    nom_pluriel = CharField(_('nom (au pluriel)'), max_length=110, blank=True,
                            help_text=PLURAL_MSG)
    classement = SmallIntegerField(default=1, db_index=True)

    def pluriel(self):
        return calc_pluriel(self)

    class Meta(object):
        verbose_name = ungettext_lazy(
            'caractéristique d’élément de programme',
            'caractéristiques d’élément de programme',
            1)
        verbose_name_plural = ungettext_lazy(
            'caractéristique d’élément de programme',
            'caractéristiques d’élément de programme',
            2)
        ordering = ('nom',)
        app_label = 'libretto'

    def __str__(self):
        return self.nom

    @staticmethod
    def autocomplete_search_fields():
        return (
            'nom__icontains',
            'nom_pluriel__icontains',
        )

@python_2_unicode_compatible
class ElementDeProgramme(AutoriteModel):
    evenement = ForeignKey('Evenement', related_name='programme',
                           db_index=True, verbose_name=_('événement'))
    oeuvre = ForeignKey('Oeuvre', related_name='elements_de_programme',
                        verbose_name=_('œuvre'), blank=True, null=True,
                        db_index=True, on_delete=PROTECT)
    autre = CharField(max_length=500, blank=True, db_index=True)
    caracteristiques = ManyToManyField(
        CaracteristiqueDElementDeProgramme,
        related_name='elements_de_programme', blank=True, null=True,
        verbose_name=_('caractéristiques'))
    NUMEROTATIONS = (
        ('O', _('Numéros')),  # O pour Ordered
        ('B', _('Numéros entre crochets (supposition)')),  # B pour Brackets
        ('U', _('Puce')),  # U pour Unordered
        ('E', _('Absente (entracte, etc)')),  # E pour Empty
    )
    numerotation = CharField(
        _('numérotation'), choices=NUMEROTATIONS, max_length=1, default='O')
    position = PositiveSmallIntegerField(_('Position'))
    # TODO: Quand les nested inlines seront possibles avec Django, remplacer
    # ceci par un GenericRelation.
    distribution = ManyToManyField(
        ElementDeDistribution, related_name='elements_de_programme',
        blank=True, null=True)
    personnels = ManyToManyField('Personnel', blank=True, null=True,
                                 related_name='elements_de_programme')

    class Meta(object):
        verbose_name = ungettext_lazy('élément de programme',
                                      'éléments de programme', 1)
        verbose_name_plural = ungettext_lazy('élément de programme',
                                             'éléments de programme', 2)
        ordering = ('position', 'oeuvre')
        app_label = 'libretto'

    def calc_caracteristiques(self):
        if self.pk is None:
            return ''
        return str_list(smart_text(c) for c in self.caracteristiques.all())
    calc_caracteristiques.allow_tags = True
    calc_caracteristiques.short_description = _('caractéristiques')

    @property
    @model_method_cached(24 * 60 * 60, b'programmes')
    def numero(self):
        numerotations_exclues = ('U', 'E',)
        if self.numerotation in numerotations_exclues:
            return ''
        return self.evenement.programme.exclude(Q(position__gt=self.position)
                           | Q(numerotation__in=numerotations_exclues)).count()

    @model_method_cached(24 * 60 * 60, b'programmes')
    def html(self, tags=True):
        has_pk = self.pk is not None

        distribution = self.distribution.prefetch()
        if has_pk and distribution:
            distribution = distribution.html(tags=tags)
            add_distribution = True
        else:
            distribution = ''
            add_distribution = False

        if self.oeuvre:
            out = self.oeuvre.html(tags)
        elif self.autre:
            out = self.autre
        elif distribution:
            out = distribution
            add_distribution = False
        else:
            warnings.warn('Il manque des champs dans <%(class)s pk=%(pk)s>' %
                          {'class': self.__class__.__name__, 'pk': self.pk})
            return ''

        caracteristiques = self.calc_caracteristiques()
        if caracteristiques:
            out += ' [' + caracteristiques + ']'

        if add_distribution:
            out += '. — ' + distribution

        return out
    html.short_description = _('rendu HTML')
    html.allow_tags = True

    def __str__(self):
        return strip_tags(self.html(False))

    @staticmethod
    def autocomplete_search_fields():
        return (
            'oeuvre__prefixe_titre__icontains', 'oeuvre__titre__icontains',
            'oeuvre__prefixe_titre_secondaire__icontains',
            'oeuvre__titre_secondaire__icontains',
            'oeuvre__genre__nom__icontains',
        )

class EvenementQuerySet(PublishedQuerySet):
    def yearly_counts(self):
        return get_model('libretto', 'AncrageSpatioTemporel').objects.filter(
            Q(evenements_debuts__in=self) | Q(evenements_fins__in=self)) \
            .extra({'year': connection.ops.date_trunc_sql('year', 'date')}) \
            .values('year').annotate(count=Count('evenements_debuts')) \
            .order_by('year')

class EvenementManager(PublishedManager):
    def get_query_set(self):
        return EvenementQuerySet(self.model, using=self._db)

    def yearly_counts(self):
        return self.get_query_set().yearly_counts()

@python_2_unicode_compatible
class Evenement(AutoriteModel):
    ancrage_debut = OneToOneField(
        'AncrageSpatioTemporel', related_name='evenements_debuts',
        db_index=True, on_delete=PROTECT)
    ancrage_fin = OneToOneField(
        'AncrageSpatioTemporel', related_name='evenements_fins', blank=True,
        null=True, db_index=True, on_delete=PROTECT)
    relache = BooleanField(verbose_name='relâche', db_index=True)
    circonstance = CharField(max_length=500, blank=True, db_index=True)
    distribution = GenericRelation(ElementDeDistribution)

    objects = EvenementManager()

    class Meta(object):
        verbose_name = ungettext_lazy('événement', 'événements', 1)
        verbose_name_plural = ungettext_lazy('événement', 'événements', 2)
        ordering = ('ancrage_debut',)
        app_label = 'libretto'
        permissions = (('can_change_status', _('Peut changer l’état')),)

    @permalink
    def get_absolute_url(self):
        return 'evenement_pk', (self.pk,)

    def permalien(self):
        return self.get_absolute_url()

    def link(self):
        return href(self.get_absolute_url(), smart_text(self))
    link.short_description = _('lien')
    link.allow_tags = True

    def sources_by_type(self):
        sources = OrderedDefaultDict()
        for source in self.sources.select_related('type'):
            sources[source.type].append(source)
        return sources.items()

    def get_meta_name(self, tags=False):
        if self.circonstance:
            out = self.circonstance
        else:
            distribution = self.distribution.prefetch()
            if distribution:
                out = distribution.html(tags=tags)
            else:
                programme = self.programme.all()
                if programme.exists():
                    element = programme[0]
                    out = element.oeuvre or element.autre
                else:
                    return ''
        return microdata(out, 'summary', tags=tags)

    def html(self, tags=True):
        relache = ''
        circonstance = ''
        if self.circonstance:
            circonstance = hlp(self.circonstance, ugettext('circonstance'),
                               tags)
        if self.relache:
            relache = microdata(ugettext('Relâche'), 'eventType', tags=tags)

        lieu = microdata(self.ancrage_debut.calc_lieu(tags), 'location',
                         tags=tags)

        return str_list((lieu, circonstance,
                         self.ancrage_debut.calc_heure(), relache))

    html.short_description = _('rendu HTML')
    html.allow_tags = True

    def has_program(self):
        return self.relache or self.programme.exists()
    has_program.short_description = _('programme')
    has_program.boolean = True

    def has_source(self):
        return self.sources.exists()
    has_source.short_description = _('source')
    has_source.boolean = True
    has_source.admin_order_field = 'sources'

    def __str__(self):
        out = self.ancrage_debut.calc_date(False)
        out = capfirst(out)
        out += '\u00A0> ' + self.html(False)
        return strip_tags(out)

    @staticmethod
    def autocomplete_search_fields():
        return (
            'circonstance__icontains',
            'ancrage_debut__lieu__nom__icontains',
            'ancrage_debut__lieu__parent__nom__icontains',
            'ancrage_debut__date__icontains',
            'ancrage_debut__heure__icontains',
            'ancrage_debut__lieu_approx__icontains',
            'ancrage_debut__date_approx__icontains',
            'ancrage_debut__heure_approx__icontains',
        )
<commit_msg>Corrige un bug d'enregistrement.<commit_after># coding: utf-8

from __future__ import unicode_literals
import warnings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.generic import GenericForeignKey, \
                                                GenericRelation
from django.core.exceptions import ValidationError
from django.db import connection
from django.db.models import CharField, ForeignKey, ManyToManyField, \
    OneToOneField, BooleanField, PositiveSmallIntegerField, permalink, Q, \
    PositiveIntegerField, get_model, SmallIntegerField, PROTECT, Count
from django.utils.encoding import python_2_unicode_compatible, smart_text
from django.utils.html import strip_tags
from django.utils.translation import ungettext_lazy
from cache_tools import model_method_cached, cached_ugettext as ugettext, \
    cached_ugettext_lazy as _
from .common import CommonModel, AutoriteModel, LOWER_MSG, PLURAL_MSG, \
    calc_pluriel, CommonQuerySet, CommonManager, OrderedDefaultDict, \
    PublishedManager, PublishedQuerySet
from .functions import capfirst, str_list, str_list_w_last, href, hlp, \
    microdata

__all__ = (b'ElementDeDistribution', b'CaracteristiqueDElementDeProgramme',
           b'ElementDeProgramme', b'Evenement')

class ElementDeDistributionQuerySet(CommonQuerySet):
    def individus(self):
        return get_model('libretto', 'Individu').objects.filter(
            pk__in=self.values_list('individus', flat=True))

    def evenements(self):
        distributions_evenements = self.filter(
            content_type__app_label='libretto',
            content_type__model='evenement')
        pk_list = distributions_evenements.values_list('object_id', flat=True)
        return Evenement.objects.filter(Q(pk__in=pk_list)
                              | Q(programme__distribution__in=self)).distinct()

    def prefetch(self):
        return self.select_related(
            'pupitre', 'pupitre__partie',
            'profession').prefetch_related('individus')

    def html(self, tags=True):
        return ', '.join(e.html(tags=tags) for e in self)

class ElementDeDistributionManager(CommonManager):
    use_for_related_fields = True

    def get_query_set(self):
        return ElementDeDistributionQuerySet(self.model, using=self._db)

    def individus(self):
        return self.get_query_set().individus()

    def evenements(self):
        return self.get_query_set().evenements()

    def prefetch(self):
        return self.get_query_set().prefetch()

    def html(self, tags=True):
        return self.get_query_set().html(tags=tags)

@python_2_unicode_compatible
class ElementDeDistribution(CommonModel):
    individus = ManyToManyField('Individu', verbose_name=_('individus'),
                                related_name='elements_de_distribution')
    pupitre = ForeignKey(
        'Pupitre', verbose_name=_('pupitre'), null=True, blank=True,
        related_name='elements_de_distribution', on_delete=PROTECT)
    profession = ForeignKey(
        'Profession', verbose_name=_('profession'), null=True, blank=True,
        related_name='elements_de_distribution', on_delete=PROTECT)
    content_type = ForeignKey(ContentType, null=True, on_delete=PROTECT)
    object_id = PositiveIntegerField(null=True)
    content_object = GenericForeignKey()

    objects = ElementDeDistributionManager()

    class Meta(object):
        verbose_name = ungettext_lazy('élément de distribution',
                                      'éléments de distribution', 1)
        verbose_name_plural = ungettext_lazy('élément de distribution',
                                             'éléments de distribution', 2)
        ordering = ('pupitre',)
        app_label = 'libretto'

    def __str__(self):
        return self.html(tags=False)

    def html(self, tags=True):
        out = ''
        if self.pk:
            individus = self.individus.all()
            out += str_list_w_last(individu.html(tags=tags)
                                   for individu in individus)

        if self.pupitre:
            out += ' [' + self.pupitre.partie.link() + ']'
        elif self.profession:
            out += ' [' + self.profession.link() + ']'

        if not tags:
            return strip_tags(out)
        return out

    def related_label(self):
        return self.get_change_link()

    @permalink
    def get_change_url(self):
        return 'admin:libretto_elementdedistribution_change', (self.pk,)

    def get_change_link(self):
        return href(self.get_change_url(), smart_text(self), new_tab=True)

    def clean(self):
        if self.pupitre and self.profession:
            raise ValidationError(_('Vous ne pouvez remplir à la fois '
                                    '« Pupitre » et « Profession ».'))

    @staticmethod
    def autocomplete_search_fields():
        return (
            'pupitre__partie__nom__icontains',
            'individus__nom__icontains',
            'individus__pseudonyme__icontains',
        )

@python_2_unicode_compatible
class CaracteristiqueDElementDeProgramme(CommonModel):
    nom = CharField(_('nom'), max_length=100, help_text=LOWER_MSG, unique=True,
                    db_index=True)
    nom_pluriel = CharField(_('nom (au pluriel)'), max_length=110, blank=True,
                            help_text=PLURAL_MSG)
    classement = SmallIntegerField(default=1, db_index=True)

    def pluriel(self):
        return calc_pluriel(self)

    class Meta(object):
        verbose_name = ungettext_lazy(
            'caractéristique d’élément de programme',
            'caractéristiques d’élément de programme',
            1)
        verbose_name_plural = ungettext_lazy(
            'caractéristique d’élément de programme',
            'caractéristiques d’élément de programme',
            2)
        ordering = ('nom',)
        app_label = 'libretto'

    def __str__(self):
        return self.nom

    @staticmethod
    def autocomplete_search_fields():
        return (
            'nom__icontains',
            'nom_pluriel__icontains',
        )

@python_2_unicode_compatible
class ElementDeProgramme(AutoriteModel):
    evenement = ForeignKey('Evenement', related_name='programme',
                           db_index=True, verbose_name=_('événement'))
    oeuvre = ForeignKey('Oeuvre', related_name='elements_de_programme',
                        verbose_name=_('œuvre'), blank=True, null=True,
                        db_index=True, on_delete=PROTECT)
    autre = CharField(max_length=500, blank=True, db_index=True)
    caracteristiques = ManyToManyField(
        CaracteristiqueDElementDeProgramme,
        related_name='elements_de_programme', blank=True, null=True,
        verbose_name=_('caractéristiques'))
    NUMEROTATIONS = (
        ('O', _('Numéros')),  # O pour Ordered
        ('B', _('Numéros entre crochets (supposition)')),  # B pour Brackets
        ('U', _('Puce')),  # U pour Unordered
        ('E', _('Absente (entracte, etc)')),  # E pour Empty
    )
    numerotation = CharField(
        _('numérotation'), choices=NUMEROTATIONS, max_length=1, default='O')
    position = PositiveSmallIntegerField(_('Position'))
    # TODO: Quand les nested inlines seront possibles avec Django, remplacer
    # ceci par un GenericRelation.
    distribution = ManyToManyField(
        ElementDeDistribution, related_name='elements_de_programme',
        blank=True, null=True)
    personnels = ManyToManyField('Personnel', blank=True, null=True,
                                 related_name='elements_de_programme')

    class Meta(object):
        verbose_name = ungettext_lazy('élément de programme',
                                      'éléments de programme', 1)
        verbose_name_plural = ungettext_lazy('élément de programme',
                                             'éléments de programme', 2)
        ordering = ('position', 'oeuvre')
        app_label = 'libretto'

    def calc_caracteristiques(self):
        if self.pk is None:
            return ''
        return str_list(smart_text(c) for c in self.caracteristiques.all())
    calc_caracteristiques.allow_tags = True
    calc_caracteristiques.short_description = _('caractéristiques')

    @property
    @model_method_cached(24 * 60 * 60, b'programmes')
    def numero(self):
        numerotations_exclues = ('U', 'E',)
        if self.numerotation in numerotations_exclues:
            return ''
        return self.evenement.programme.exclude(Q(position__gt=self.position)
                           | Q(numerotation__in=numerotations_exclues)).count()

    @model_method_cached(24 * 60 * 60, b'programmes')
    def html(self, tags=True):
        has_pk = self.pk is not None

        distribution = ''
        add_distribution = False
        if has_pk:
            distribution = self.distribution.prefetch()
            if distribution:
                distribution = distribution.html(tags=tags)
                add_distribution = True

        if self.oeuvre:
            out = self.oeuvre.html(tags)
        elif self.autre:
            out = self.autre
        elif distribution:
            out = distribution
            add_distribution = False
        else:
            warnings.warn('Il manque des champs dans <%(class)s pk=%(pk)s>' %
                          {'class': self.__class__.__name__, 'pk': self.pk})
            return ''

        caracteristiques = self.calc_caracteristiques()
        if caracteristiques:
            out += ' [' + caracteristiques + ']'

        if add_distribution:
            out += '. — ' + distribution

        return out
    html.short_description = _('rendu HTML')
    html.allow_tags = True

    def __str__(self):
        return strip_tags(self.html(False))

    @staticmethod
    def autocomplete_search_fields():
        return (
            'oeuvre__prefixe_titre__icontains', 'oeuvre__titre__icontains',
            'oeuvre__prefixe_titre_secondaire__icontains',
            'oeuvre__titre_secondaire__icontains',
            'oeuvre__genre__nom__icontains',
        )

class EvenementQuerySet(PublishedQuerySet):
    def yearly_counts(self):
        return get_model('libretto', 'AncrageSpatioTemporel').objects.filter(
            Q(evenements_debuts__in=self) | Q(evenements_fins__in=self)) \
            .extra({'year': connection.ops.date_trunc_sql('year', 'date')}) \
            .values('year').annotate(count=Count('evenements_debuts')) \
            .order_by('year')

class EvenementManager(PublishedManager):
    def get_query_set(self):
        return EvenementQuerySet(self.model, using=self._db)

    def yearly_counts(self):
        return self.get_query_set().yearly_counts()

@python_2_unicode_compatible
class Evenement(AutoriteModel):
    ancrage_debut = OneToOneField(
        'AncrageSpatioTemporel', related_name='evenements_debuts',
        db_index=True, on_delete=PROTECT)
    ancrage_fin = OneToOneField(
        'AncrageSpatioTemporel', related_name='evenements_fins', blank=True,
        null=True, db_index=True, on_delete=PROTECT)
    relache = BooleanField(verbose_name='relâche', db_index=True)
    circonstance = CharField(max_length=500, blank=True, db_index=True)
    distribution = GenericRelation(ElementDeDistribution)

    objects = EvenementManager()

    class Meta(object):
        verbose_name = ungettext_lazy('événement', 'événements', 1)
        verbose_name_plural = ungettext_lazy('événement', 'événements', 2)
        ordering = ('ancrage_debut',)
        app_label = 'libretto'
        permissions = (('can_change_status', _('Peut changer l’état')),)

    @permalink
    def get_absolute_url(self):
        return 'evenement_pk', (self.pk,)

    def permalien(self):
        return self.get_absolute_url()

    def link(self):
        return href(self.get_absolute_url(), smart_text(self))
    link.short_description = _('lien')
    link.allow_tags = True

    def sources_by_type(self):
        sources = OrderedDefaultDict()
        for source in self.sources.select_related('type'):
            sources[source.type].append(source)
        return sources.items()

    def get_meta_name(self, tags=False):
        if self.circonstance:
            out = self.circonstance
        else:
            distribution = self.distribution.prefetch()
            if distribution:
                out = distribution.html(tags=tags)
            else:
                programme = self.programme.all()
                if programme.exists():
                    element = programme[0]
                    out = element.oeuvre or element.autre
                else:
                    return ''
        return microdata(out, 'summary', tags=tags)

    def html(self, tags=True):
        relache = ''
        circonstance = ''
        if self.circonstance:
            circonstance = hlp(self.circonstance, ugettext('circonstance'),
                               tags)
        if self.relache:
            relache = microdata(ugettext('Relâche'), 'eventType', tags=tags)

        lieu = microdata(self.ancrage_debut.calc_lieu(tags), 'location',
                         tags=tags)

        return str_list((lieu, circonstance,
                         self.ancrage_debut.calc_heure(), relache))

    html.short_description = _('rendu HTML')
    html.allow_tags = True

    def has_program(self):
        return self.relache or self.programme.exists()
    has_program.short_description = _('programme')
    has_program.boolean = True

    def has_source(self):
        return self.sources.exists()
    has_source.short_description = _('source')
    has_source.boolean = True
    has_source.admin_order_field = 'sources'

    def __str__(self):
        out = self.ancrage_debut.calc_date(False)
        out = capfirst(out)
        out += '\u00A0> ' + self.html(False)
        return strip_tags(out)

    @staticmethod
    def autocomplete_search_fields():
        return (
            'circonstance__icontains',
            'ancrage_debut__lieu__nom__icontains',
            'ancrage_debut__lieu__parent__nom__icontains',
            'ancrage_debut__date__icontains',
            'ancrage_debut__heure__icontains',
            'ancrage_debut__lieu_approx__icontains',
            'ancrage_debut__date_approx__icontains',
            'ancrage_debut__heure_approx__icontains',
        )
<|endoftext|>


<commit_before>import genomepy
import gzip
import os
import pytest
import requests

travis = "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true"

def validate_gzipped_gtf(fname):
    assert os.path.exists(fname)
    with gzip.open(fname, "r") as f:
        for line in f:
            line = line.decode()
            if line.startswith("#"):
                continue
            vals = line.split("\t")
            assert 9 == len(vals)
            int(vals[3]), int(vals[4])
            break

def validate_gzipped_bed(fname):
    assert os.path.exists(fname)
    with gzip.open(fname, "r") as f:
        for line in f:
            line = line.decode()
            if line.startswith("#"):
                continue
            vals = line.split("\t")
            assert 12 == len(vals)
            int(vals[1]), int(vals[2])
            break

@pytest.fixture(scope="module")
def p():
    return genomepy.provider.EnsemblProvider()

def test_ensemblprovider__init__(p):
    p2 = genomepy.provider.ProviderBase().create("Ensembl")
    assert p.name == p2.name == "Ensembl"
    assert p.taxid_fields == ["taxonomy_id"]

def test__request_json(p):
    divisions = p._request_json("info/divisions?")
    assert isinstance(divisions, list)
    assert "EnsemblVertebrates" in divisions

    # test not r.ok
    with pytest.raises(requests.exceptions.HTTPError):
        p._request_json("error")

def test__get_genomes(p):
    assert isinstance(p.genomes, dict)
    assert "KH" in p.genomes
    genome = p.genomes["KH"]
    assert isinstance(genome, dict)
    for field in p.accession_fields + p.taxid_fields + p.description_fields:
        assert field in genome
    assert genome["taxonomy_id"] == 7719

def test_genome_info_tuple(p):
    t = p._genome_info_tuple("KH")
    assert isinstance(t, tuple)
    assert t[2:4] == ("Ciona intestinalis", "7719")

def test_get_version(p):
    # note: this test will break every time ensembl releases a new version
    ftp_site = "http://ftp.ensembl.org/pub"
    v = p.get_version(ftp_site)
    assert v == "100"

    if not travis:
        ftp_site = "ftp://ftp.ensemblgenomes.org/pub"
        v = p.get_version(ftp_site)
        assert v == "47"

def test_get_genome_download_link(p):
    if not travis:
        # non vertebrate: soft masked
        link = p.get_genome_download_link("TAIR10", mask="soft", **{"version": 46})
        assert (
            link
            == "ftp://ftp.ensemblgenomes.org/pub/plants/release-46/"
            + "fasta/arabidopsis_thaliana/dna/Arabidopsis_thaliana.TAIR10.dna_sm.toplevel.fa.gz"
        )

    # vertebrate with primary assembly: unmasked
    link = p.get_genome_download_link("GRCz11", mask="none", **{"version": 98})
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/fasta/"
        + "danio_rerio/dna/Danio_rerio.GRCz11.dna.primary_assembly.fa.gz"
    )

    # vertebrate with primary assembly: hard masked and toplevel only
    link = p.get_genome_download_link(
        "GRCz11", mask="hard", **{"version": 98, "toplevel": True}
    )
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/fasta/"
        + "danio_rerio/dna/Danio_rerio.GRCz11.dna_rm.toplevel.fa.gz"
    )

    # vertebrate: any version
    p.version = None
    link = p.get_genome_download_link("GRCz11", **{"toplevel": True})
    for substring in [
        "http://ftp.ensembl.org/pub/release-",
        "/fasta/danio_rerio/dna/Danio_rerio.GRCz11.dna_sm.toplevel.fa.gz",
    ]:
        assert substring in link

def test_get_annotation_download_link(p):
    if not travis:
        # non vertebrate
        link = p.get_annotation_download_link("TAIR10", **{"version": 46})
        assert (
            link
            == "ftp://ftp.ensemblgenomes.org/pub/plants/release-46/"
            + "gtf/arabidopsis_thaliana/Arabidopsis_thaliana.TAIR10.46.gtf.gz"
        )

    # vertebrate
    link = p.get_annotation_download_link("GRCz11", **{"version": 98})
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/gtf/"
        + "danio_rerio/Danio_rerio.GRCz11.98.gtf.gz"
    )

    # vertebrate: any version
    p.version = None
    link = p.get_annotation_download_link("GRCz11")
    for substring in [
        "http://ftp.ensembl.org/pub/release-",
        "/gtf/danio_rerio/Danio_rerio.GRCz11.",
        ".gtf.gz",
    ]:
        assert substring in link
<commit_msg>updated ensembl version<commit_after>import genomepy
import gzip
import os
import pytest
import requests

travis = "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true"

def validate_gzipped_gtf(fname):
    assert os.path.exists(fname)
    with gzip.open(fname, "r") as f:
        for line in f:
            line = line.decode()
            if line.startswith("#"):
                continue
            vals = line.split("\t")
            assert 9 == len(vals)
            int(vals[3]), int(vals[4])
            break

def validate_gzipped_bed(fname):
    assert os.path.exists(fname)
    with gzip.open(fname, "r") as f:
        for line in f:
            line = line.decode()
            if line.startswith("#"):
                continue
            vals = line.split("\t")
            assert 12 == len(vals)
            int(vals[1]), int(vals[2])
            break

@pytest.fixture(scope="module")
def p():
    return genomepy.provider.EnsemblProvider()

def test_ensemblprovider__init__(p):
    p2 = genomepy.provider.ProviderBase().create("Ensembl")
    assert p.name == p2.name == "Ensembl"
    assert p.taxid_fields == ["taxonomy_id"]

def test__request_json(p):
    divisions = p._request_json("info/divisions?")
    assert isinstance(divisions, list)
    assert "EnsemblVertebrates" in divisions

    # test not r.ok
    with pytest.raises(requests.exceptions.HTTPError):
        p._request_json("error")

def test__get_genomes(p):
    assert isinstance(p.genomes, dict)
    assert "KH" in p.genomes
    genome = p.genomes["KH"]
    assert isinstance(genome, dict)
    for field in p.accession_fields + p.taxid_fields + p.description_fields:
        assert field in genome
    assert genome["taxonomy_id"] == 7719

def test_genome_info_tuple(p):
    t = p._genome_info_tuple("KH")
    assert isinstance(t, tuple)
    assert t[2:4] == ("Ciona intestinalis", "7719")

def test_get_version(p):
    # note: this test will break every time ensembl releases a new version
    ftp_site = "http://ftp.ensembl.org/pub"
    v = p.get_version(ftp_site)
    assert v == "101"

    if not travis:
        ftp_site = "ftp://ftp.ensemblgenomes.org/pub"
        v = p.get_version(ftp_site)
        assert v == "48"

def test_get_genome_download_link(p):
    if not travis:
        # non vertebrate: soft masked
        link = p.get_genome_download_link("TAIR10", mask="soft", **{"version": 46})
        assert (
            link
            == "ftp://ftp.ensemblgenomes.org/pub/plants/release-46/"
            + "fasta/arabidopsis_thaliana/dna/Arabidopsis_thaliana.TAIR10.dna_sm.toplevel.fa.gz"
        )

    # vertebrate with primary assembly: unmasked
    link = p.get_genome_download_link("GRCz11", mask="none", **{"version": 98})
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/fasta/"
        + "danio_rerio/dna/Danio_rerio.GRCz11.dna.primary_assembly.fa.gz"
    )

    # vertebrate with primary assembly: hard masked and toplevel only
    link = p.get_genome_download_link(
        "GRCz11", mask="hard", **{"version": 98, "toplevel": True}
    )
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/fasta/"
        + "danio_rerio/dna/Danio_rerio.GRCz11.dna_rm.toplevel.fa.gz"
    )

    # vertebrate: any version
    p.version = None
    link = p.get_genome_download_link("GRCz11", **{"toplevel": True})
    for substring in [
        "http://ftp.ensembl.org/pub/release-",
        "/fasta/danio_rerio/dna/Danio_rerio.GRCz11.dna_sm.toplevel.fa.gz",
    ]:
        assert substring in link

def test_get_annotation_download_link(p):
    if not travis:
        # non vertebrate
        link = p.get_annotation_download_link("TAIR10", **{"version": 46})
        assert (
            link
            == "ftp://ftp.ensemblgenomes.org/pub/plants/release-46/"
            + "gtf/arabidopsis_thaliana/Arabidopsis_thaliana.TAIR10.46.gtf.gz"
        )

    # vertebrate
    link = p.get_annotation_download_link("GRCz11", **{"version": 98})
    assert (
        link
        == "http://ftp.ensembl.org/pub/release-98/gtf/"
        + "danio_rerio/Danio_rerio.GRCz11.98.gtf.gz"
    )

    # vertebrate: any version
    p.version = None
    link = p.get_annotation_download_link("GRCz11")
    for substring in [
        "http://ftp.ensembl.org/pub/release-",
        "/gtf/danio_rerio/Danio_rerio.GRCz11.",
        ".gtf.gz",
    ]:
        assert substring in link
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Fixes a MySQL dump made with the right format so it can be directly
imported to a new PostgreSQL database.

Dump using:
mysqldump --compatible=postgresql --default-character-set=utf8 -r databasename.mysql -u root databasename
"""

import re
import sys
import os
import time
import subprocess

def parse(input_filename, output_filename):
    "Feed it a file, and it'll output a fixed one"

    # State storage
    if input_filename == "-":
        num_lines = -1
    else:
        num_lines = int(subprocess.check_output(["wc", "-l", input_filename]).strip().split()[0])
    tables = {}
    current_table = None
    creation_lines = []
    enum_types = []
    foreign_key_lines = []
    fulltext_key_lines = []
    sequence_lines = []
    cast_lines = []
    num_inserts = 0
    started = time.time()

    # Open output file and write header. Logging file handle will be stdout
    # unless we're writing output to stdout, in which case NO PROGRESS FOR YOU.
    if output_filename == "-":
        output = sys.stdout
        logging = open(os.devnull, "w")
    else:
        output = open(output_filename, "w")
        logging = sys.stdout

    if input_filename == "-":
        input_fh = sys.stdin
    else:
        input_fh = open(input_filename)

    output.write("-- Converted by db_converter\n")
    output.write("START TRANSACTION;\n")
    output.write("SET standard_conforming_strings=off;\n")
    output.write("SET escape_string_warning=off;\n")
    output.write("SET CONSTRAINTS ALL DEFERRED;\n\n")

    for i, line in enumerate(input_fh):
        time_taken = time.time() - started
        percentage_done = (i+1) / float(num_lines)
        secs_left = (time_taken / percentage_done) - time_taken
        logging.write("\rLine %i (of %s: %.2f%%) [%s tables] [%s inserts] [ETA: %i min %i sec]" % (
            i + 1,
            num_lines,
            ((i+1)/float(num_lines))*100,
            len(tables),
            num_inserts,
            secs_left // 60,
            secs_left % 60,
        ))
        logging.flush()
        line = line.decode("utf8").strip().replace(r"\\", "WUBWUBREALSLASHWUB").replace(r"\'", "''").replace("WUBWUBREALSLASHWUB", r"\\")
        # Ignore comment lines
        if line.startswith("--") or line.startswith("/*") or line.startswith("LOCK TABLES") or line.startswith("DROP TABLE") or line.startswith("UNLOCK TABLES") or not line:
            continue

        # Outside of anything handling
        if current_table is None:
            # Start of a table creation statement?
            if line.startswith("CREATE TABLE"):
                current_table = line.split('"')[1]
                tables[current_table] = {"columns": []}
                creation_lines = []
            # Inserting data into a table?
            elif line.startswith("INSERT INTO"):
                output.write(line.encode("utf8").replace("'0000-00-00 00:00:00'", "NULL") + "\n")
                num_inserts += 1
            # ???
            elif line.startswith("DROP TABLE"):
                output.write(line.encode("utf8") + "\n")
            else:
                print "\n ! Unknown line in main body: %s" % line

        # Inside-create-statement handling
        else:
            # Is it a column?
            if line.startswith('"'):
                useless, name, definition = line.strip(",").split('"',2)
                try:
                    type, extra = definition.strip().split(" ", 1)

                    # This must be a tricky enum
                    if ')' in extra:
                        type, extra = definition.strip().split(")")

                except ValueError:
                    type = definition.strip()
                    extra = ""
                extra = re.sub("CHARACTER SET [\w\d]+\s*", "", extra.replace("unsigned", ""))
                extra = re.sub("COLLATE [\w\d]+\s*", "", extra.replace("unsigned", ""))

                # See if it needs type conversion
                final_type = None
                set_sequence = None
                if type == "tinyint(1)":
                    type = "int4"
                    set_sequence = True
                    final_type = "boolean"
                elif type.startswith("int("):
                    type = "integer"
                    set_sequence = True
                elif type.startswith("bigint("):
                    type = "bigint"
                    set_sequence = True
                elif type == "longtext":
                    type = "text"
                elif type == "mediumtext":
                    type = "text"
                elif type == "tinytext":
                    type = "text"
                elif type.startswith("varchar("):
                    size = int(type.split("(")[1].rstrip(")"))
                    type = "varchar(%s)" % (size * 2)
                elif type.startswith("smallint("):
                    type = "int2"
                    set_sequence = True
                elif type == "datetime":
                    type = "timestamp with time zone"
                elif type == "double":
                    type = "double precision"
                elif type == "blob":
                    type = "bytea"
                elif type.startswith("enum(") or type.startswith("set("):

                    types_str = type.split("(")[1].rstrip(")").rstrip('"')
                    types_arr = [type_str.strip('\'') for type_str in types_str.split(",")]

                    # Considered using values to make a name, but its dodgy
                    # enum_name = '_'.join(types_arr)
                    enum_name = "{0}_{1}".format(current_table, name)

                    if enum_name not in enum_types:
                        output.write("CREATE TYPE {0} AS ENUM ({1}); \n".format(enum_name, types_str));
                        enum_types.append(enum_name)

                    type = enum_name

                if final_type:
                    cast_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" DROP DEFAULT, ALTER COLUMN \"%s\" TYPE %s USING CAST(\"%s\" as %s)" % (current_table, name, name, final_type, name, final_type))
                # ID fields need sequences [if they are integers?]
                if name == "id" and set_sequence is True:
                    sequence_lines.append("CREATE SEQUENCE %s_id_seq" % (current_table))
                    sequence_lines.append("SELECT setval('%s_id_seq', max(id)) FROM %s" % (current_table, current_table))
                    sequence_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"id\" SET DEFAULT nextval('%s_id_seq')" % (current_table, current_table))
                # Record it
                creation_lines.append('"%s" %s %s' % (name, type, extra))
                tables[current_table]['columns'].append((name, type, extra))
            # Is it a constraint or something?
            elif line.startswith("PRIMARY KEY"):
                creation_lines.append(line.rstrip(","))
            elif line.startswith("CONSTRAINT"):
                foreign_key_lines.append("ALTER TABLE \"%s\" ADD CONSTRAINT %s DEFERRABLE INITIALLY DEFERRED" % (current_table, line.split("CONSTRAINT")[1].strip().rstrip(",")))
                foreign_key_lines.append("CREATE INDEX ON \"%s\" %s" % (current_table, line.split("FOREIGN KEY")[1].split("REFERENCES")[0].strip().rstrip(",")))
            elif line.startswith("UNIQUE KEY"):
                creation_lines.append("UNIQUE (%s)" % line.split("(")[1].split(")")[0])
            elif line.startswith("FULLTEXT KEY"):

                fulltext_keys = " || ' ' || ".join( line.split('(')[-1].split(')')[0].replace('"', '').split(',') )
                fulltext_key_lines.append("CREATE INDEX ON %s USING gin(to_tsvector('english', %s))" % (current_table, fulltext_keys))

            elif line.startswith("KEY"):
                pass
            # Is it the end of the table?
            elif line == ");":
                output.write("CREATE TABLE \"%s\" (\n" % current_table)
                for i, line in enumerate(creation_lines):
                    output.write("    %s%s\n" % (line, "," if i != (len(creation_lines) - 1) else ""))
                output.write(');\n\n')
                current_table = None
            # ???
            else:
                print "\n ! Unknown line inside table creation: %s" % line

    # Finish file
    output.write("\n-- Post-data save --\n")
    output.write("COMMIT;\n")
    output.write("START TRANSACTION;\n")

    # Write typecasts out
    output.write("\n-- Typecasts --\n")
    for line in cast_lines:
        output.write("%s;\n" % line)

    # Write FK constraints out
    output.write("\n-- Foreign keys --\n")
    for line in foreign_key_lines:
        output.write("%s;\n" % line)

    # Write sequences out
    output.write("\n-- Sequences --\n")
    for line in sequence_lines:
        output.write("%s;\n" % line)

    # Write full-text indexkeyses out
    output.write("\n-- Full Text keys --\n")
    for line in fulltext_key_lines:
        output.write("%s;\n" % line)

    # Finish file
    output.write("\n")
    output.write("COMMIT;\n")
    print ""

if __name__ == "__main__":
    parse(sys.argv[1], sys.argv[2])
<commit_msg>don't skip DROP TABLE lines so we can pass them through later<commit_after>#!/usr/bin/env python

"""
Fixes a MySQL dump made with the right format so it can be directly
imported to a new PostgreSQL database.

Dump using:
mysqldump --compatible=postgresql --default-character-set=utf8 -r databasename.mysql -u root databasename
"""

import re
import sys
import os
import time
import subprocess

def parse(input_filename, output_filename):
    "Feed it a file, and it'll output a fixed one"

    # State storage
    if input_filename == "-":
        num_lines = -1
    else:
        num_lines = int(subprocess.check_output(["wc", "-l", input_filename]).strip().split()[0])
    tables = {}
    current_table = None
    creation_lines = []
    enum_types = []
    foreign_key_lines = []
    fulltext_key_lines = []
    sequence_lines = []
    cast_lines = []
    num_inserts = 0
    started = time.time()

    # Open output file and write header. Logging file handle will be stdout
    # unless we're writing output to stdout, in which case NO PROGRESS FOR YOU.
    if output_filename == "-":
        output = sys.stdout
        logging = open(os.devnull, "w")
    else:
        output = open(output_filename, "w")
        logging = sys.stdout

    if input_filename == "-":
        input_fh = sys.stdin
    else:
        input_fh = open(input_filename)

    output.write("-- Converted by db_converter\n")
    output.write("START TRANSACTION;\n")
    output.write("SET standard_conforming_strings=off;\n")
    output.write("SET escape_string_warning=off;\n")
    output.write("SET CONSTRAINTS ALL DEFERRED;\n\n")

    for i, line in enumerate(input_fh):
        time_taken = time.time() - started
        percentage_done = (i+1) / float(num_lines)
        secs_left = (time_taken / percentage_done) - time_taken
        logging.write("\rLine %i (of %s: %.2f%%) [%s tables] [%s inserts] [ETA: %i min %i sec]" % (
            i + 1,
            num_lines,
            ((i+1)/float(num_lines))*100,
            len(tables),
            num_inserts,
            secs_left // 60,
            secs_left % 60,
        ))
        logging.flush()
        line = line.decode("utf8").strip().replace(r"\\", "WUBWUBREALSLASHWUB").replace(r"\'", "''").replace("WUBWUBREALSLASHWUB", r"\\")
        # Ignore comment lines
        if line.startswith("--") or line.startswith("/*") or line.startswith("LOCK TABLES") or line.startswith("UNLOCK TABLES") or not line:
            continue

        # Outside of anything handling
        if current_table is None:
            # Start of a table creation statement?
            if line.startswith("CREATE TABLE"):
                current_table = line.split('"')[1]
                tables[current_table] = {"columns": []}
                creation_lines = []
            # Inserting data into a table?
            elif line.startswith("INSERT INTO"):
                output.write(line.encode("utf8").replace("'0000-00-00 00:00:00'", "NULL") + "\n")
                num_inserts += 1
            # ???
            elif line.startswith("DROP TABLE"):
                output.write(line.encode("utf8") + "\n")
            else:
                print "\n ! Unknown line in main body: %s" % line

        # Inside-create-statement handling
        else:
            # Is it a column?
            if line.startswith('"'):
                useless, name, definition = line.strip(",").split('"',2)
                try:
                    type, extra = definition.strip().split(" ", 1)

                    # This must be a tricky enum
                    if ')' in extra:
                        type, extra = definition.strip().split(")")

                except ValueError:
                    type = definition.strip()
                    extra = ""
                extra = re.sub("CHARACTER SET [\w\d]+\s*", "", extra.replace("unsigned", ""))
                extra = re.sub("COLLATE [\w\d]+\s*", "", extra.replace("unsigned", ""))

                # See if it needs type conversion
                final_type = None
                set_sequence = None
                if type == "tinyint(1)":
                    type = "int4"
                    set_sequence = True
                    final_type = "boolean"
                elif type.startswith("int("):
                    type = "integer"
                    set_sequence = True
                elif type.startswith("bigint("):
                    type = "bigint"
                    set_sequence = True
                elif type == "longtext":
                    type = "text"
                elif type == "mediumtext":
                    type = "text"
                elif type == "tinytext":
                    type = "text"
                elif type.startswith("varchar("):
                    size = int(type.split("(")[1].rstrip(")"))
                    type = "varchar(%s)" % (size * 2)
                elif type.startswith("smallint("):
                    type = "int2"
                    set_sequence = True
                elif type == "datetime":
                    type = "timestamp with time zone"
                elif type == "double":
                    type = "double precision"
                elif type == "blob":
                    type = "bytea"
                elif type.startswith("enum(") or type.startswith("set("):

                    types_str = type.split("(")[1].rstrip(")").rstrip('"')
                    types_arr = [type_str.strip('\'') for type_str in types_str.split(",")]

                    # Considered using values to make a name, but its dodgy
                    # enum_name = '_'.join(types_arr)
                    enum_name = "{0}_{1}".format(current_table, name)

                    if enum_name not in enum_types:
                        output.write("CREATE TYPE {0} AS ENUM ({1}); \n".format(enum_name, types_str));
                        enum_types.append(enum_name)

                    type = enum_name

                if final_type:
                    cast_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" DROP DEFAULT, ALTER COLUMN \"%s\" TYPE %s USING CAST(\"%s\" as %s)" % (current_table, name, name, final_type, name, final_type))
                # ID fields need sequences [if they are integers?]
                if name == "id" and set_sequence is True:
                    sequence_lines.append("CREATE SEQUENCE %s_id_seq" % (current_table))
                    sequence_lines.append("SELECT setval('%s_id_seq', max(id)) FROM %s" % (current_table, current_table))
                    sequence_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"id\" SET DEFAULT nextval('%s_id_seq')" % (current_table, current_table))
                # Record it
                creation_lines.append('"%s" %s %s' % (name, type, extra))
                tables[current_table]['columns'].append((name, type, extra))
            # Is it a constraint or something?
            elif line.startswith("PRIMARY KEY"):
                creation_lines.append(line.rstrip(","))
            elif line.startswith("CONSTRAINT"):
                foreign_key_lines.append("ALTER TABLE \"%s\" ADD CONSTRAINT %s DEFERRABLE INITIALLY DEFERRED" % (current_table, line.split("CONSTRAINT")[1].strip().rstrip(",")))
                foreign_key_lines.append("CREATE INDEX ON \"%s\" %s" % (current_table, line.split("FOREIGN KEY")[1].split("REFERENCES")[0].strip().rstrip(",")))
            elif line.startswith("UNIQUE KEY"):
                creation_lines.append("UNIQUE (%s)" % line.split("(")[1].split(")")[0])
            elif line.startswith("FULLTEXT KEY"):

                fulltext_keys = " || ' ' || ".join( line.split('(')[-1].split(')')[0].replace('"', '').split(',') )
                fulltext_key_lines.append("CREATE INDEX ON %s USING gin(to_tsvector('english', %s))" % (current_table, fulltext_keys))

            elif line.startswith("KEY"):
                pass
            # Is it the end of the table?
            elif line == ");":
                output.write("CREATE TABLE \"%s\" (\n" % current_table)
                for i, line in enumerate(creation_lines):
                    output.write("    %s%s\n" % (line, "," if i != (len(creation_lines) - 1) else ""))
                output.write(');\n\n')
                current_table = None
            # ???
            else:
                print "\n ! Unknown line inside table creation: %s" % line

    # Finish file
    output.write("\n-- Post-data save --\n")
    output.write("COMMIT;\n")
    output.write("START TRANSACTION;\n")

    # Write typecasts out
    output.write("\n-- Typecasts --\n")
    for line in cast_lines:
        output.write("%s;\n" % line)

    # Write FK constraints out
    output.write("\n-- Foreign keys --\n")
    for line in foreign_key_lines:
        output.write("%s;\n" % line)

    # Write sequences out
    output.write("\n-- Sequences --\n")
    for line in sequence_lines:
        output.write("%s;\n" % line)

    # Write full-text indexkeyses out
    output.write("\n-- Full Text keys --\n")
    for line in fulltext_key_lines:
        output.write("%s;\n" % line)

    # Finish file
    output.write("\n")
    output.write("COMMIT;\n")
    print ""

if __name__ == "__main__":
    parse(sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before><commit_msg>add tax_ids to onchange<commit_after><|endoftext|>


<commit_before>import scipy
from scipy import ndimage
import cv2
import numpy as np
import sys
import torch
import deeplab_resnet_sketchParse_r1
from torch.autograd import Variable
import torchvision.models as models
import torch.nn.functional as F
import torch.nn as nn
from collections import OrderedDict
import os
from os import walk
import matplotlib.pyplot as plt
from docopt import docopt

docstr = """Evaluate ResNet-DeepLab with 5 branches on sketches of 11 categories (5 super categories)

Usage:
    eval_r5.py [options]

Options:
    -h, --help                  Print this message
    --visualize                 view outputs of each sketch
    --snapPrefix=<str>          Snapshot [default: NoFile]
    --testGTpath=<str>              Ground truth path prefix [default: data/sketch-dataset/test_GT/]
    --testIMpath=<str>              Sketch images path prefix [default: data/sketch-dataset/PNG_untouched/]
    --gpu0=<int>                GPU number [default: 0]
"""

args = docopt(docstr, version='v0.1')
print args

def get_iou(pred,gt):
    assert(pred.shape == gt.shape)
    gt = gt.astype(np.float32)
    pred = pred.astype(np.float32)

    max_label = 4
    count = np.zeros((max_label+1,))
    for j in range(max_label+1):
        x = np.where(pred==j)
        p_idx_j = set(zip(x[0].tolist(),x[1].tolist()))
        x = np.where(gt==j)
        GT_idx_j = set(zip(x[0].tolist(),x[1].tolist()))
        n_jj = set.intersection(p_idx_j,GT_idx_j)
        u_jj = set.union(p_idx_j,GT_idx_j)

        if len(GT_idx_j)!=0:
            count[j] = float(len(n_jj))/float(len(u_jj))

    result_class = count

    Aiou = np.sum(result_class[:])/float(len(np.unique(gt)))

    return Aiou

gpu0 = int(args['--gpu0'])
sketch_root = args['--testIMpath']
model = getattr(deeplab_resnet_sketchParse_r1,'Res_Deeplab')()
model.eval()
counter = 0
model.cuda(gpu0)
snapPrefix= args['--snapPrefix']
for iter in range(1,20):
    saved_state_dict = torch.load('snapshots/'+snapPrefix+str(iter)+'000.pth')
    if counter==0:
	print snapPrefix
    counter+=1
    model.load_state_dict(saved_state_dict)

    class_list = ['cow', 'horse']
    pytorch_list = [];
    for class_ in class_list:
        gt_path = args['--testGTpath']+class_

        img_list = next(os.walk(gt_path))[2]
        path = sketch_root + class_
        for i in img_list:

            img = cv2.imread(path+'/'+i)
            img = ndimage.grey_erosion(img[:,:,0].astype(np.uint8), size=(2,2))
            img = np.repeat(img[:,:,np.newaxis],3,2)
            gt = cv2.imread(gt_path+'/'+i, 0)
            output = model(Variable(torch.from_numpy(img[np.newaxis, :].transpose(0,3,1,2)).float(), volatile=True).cuda(gpu0))
            interp = nn.UpsamplingBilinear2d(size=(321, 321))

            if args['--visualize']:
                output_temp = interp(output[3]).cpu().data[0].numpy()
                output_temp = output_temp.transpose(1,2,0)
                output_temp = np.argmax(output_temp,axis = 2)
                plt.subplot(1,3,1)
                plt.imshow(img)
                plt.subplot(1,3,2)
                plt.imshow(gt)
                plt.subplot(1,3,3)
                plt.imshow(output_temp)
                plt.show()

            output = interp(output[3]).cpu().data[0].numpy()
            output = output.transpose(1,2,0)
            output = np.argmax(output,axis = 2)
            iou_pytorch = get_iou(output,gt)
            cv2.imwrite('test/output.png',output)
            cv2.imwrite('test/gt.png',gt)
            pytorch_list.append(iou_pytorch)

    print 'pytorch',iter, np.sum(np.asarray(pytorch_list))/len(pytorch_list)
<commit_msg>Update eval_r1.py<commit_after>import scipy
from scipy import ndimage
import cv2
import numpy as np
import sys
import torch
import deeplab_resnet_sketchParse_r1
from torch.autograd import Variable
import torchvision.models as models
import torch.nn.functional as F
import torch.nn as nn
from collections import OrderedDict
import os
from os import walk
import matplotlib.pyplot as plt
from docopt import docopt

docstr = """Evaluate ResNet-DeepLab with 5 branches on sketches of 11 categories (5 super categories)

Usage:
    eval_r5.py [options]

Options:
    -h, --help                  Print this message
    --visualize                 view outputs of each sketch
    --snapPrefix=<str>          Snapshot [default: NoFile]
    --testGTpath=<str>              Ground truth path prefix [default: data/sketch-dataset/test_GT/]
    --testIMpath=<str>              Sketch images path prefix [default: data/sketch-dataset/PNG_untouched/]
    --gpu0=<int>                GPU number [default: 0]
"""

args = docopt(docstr, version='v0.1')
print args

def get_iou(pred,gt):
    assert(pred.shape == gt.shape)
    gt = gt.astype(np.float32)
    pred = pred.astype(np.float32)

    max_label = 4
    count = np.zeros((max_label+1,))
    for j in range(max_label+1):
        x = np.where(pred==j)
        p_idx_j = set(zip(x[0].tolist(),x[1].tolist()))
        x = np.where(gt==j)
        GT_idx_j = set(zip(x[0].tolist(),x[1].tolist()))
        n_jj = set.intersection(p_idx_j,GT_idx_j)
        u_jj = set.union(p_idx_j,GT_idx_j)

        if len(GT_idx_j)!=0:
            count[j] = float(len(n_jj))/float(len(u_jj))

    result_class = count

    Aiou = np.sum(result_class[:])/float(len(np.unique(gt)))

    return Aiou

gpu0 = int(args['--gpu0'])
sketch_root = args['--testIMpath']
model = getattr(deeplab_resnet_sketchParse_r1,'Res_Deeplab')()
model.eval()
counter = 0
model.cuda(gpu0)
snapPrefix= args['--snapPrefix']
for iter in range(1,20):
    saved_state_dict = torch.load('data/snapshots/'+snapPrefix+str(iter)+'000.pth')
    if counter==0:
	print snapPrefix
    counter+=1
    model.load_state_dict(saved_state_dict)

    class_list = ['cow', 'horse']
    pytorch_list = [];
    for class_ in class_list:
        gt_path = args['--testGTpath']+class_

        img_list = next(os.walk(gt_path))[2]
        path = sketch_root + class_
        for i in img_list:

            img = cv2.imread(path+'/'+i)
            img = ndimage.grey_erosion(img[:,:,0].astype(np.uint8), size=(2,2))
            img = np.repeat(img[:,:,np.newaxis],3,2)
            gt = cv2.imread(gt_path+'/'+i, 0)
            output = model(Variable(torch.from_numpy(img[np.newaxis, :].transpose(0,3,1,2)).float(), volatile=True).cuda(gpu0))
            interp = nn.UpsamplingBilinear2d(size=(321, 321))

            if args['--visualize']:
                output_temp = interp(output[3]).cpu().data[0].numpy()
                output_temp = output_temp.transpose(1,2,0)
                output_temp = np.argmax(output_temp,axis = 2)
                plt.subplot(1,3,1)
                plt.imshow(img)
                plt.subplot(1,3,2)
                plt.imshow(gt)
                plt.subplot(1,3,3)
                plt.imshow(output_temp)
                plt.show()

            output = interp(output[3]).cpu().data[0].numpy()
            output = output.transpose(1,2,0)
            output = np.argmax(output,axis = 2)
            iou_pytorch = get_iou(output,gt)
            cv2.imwrite('test/output.png',output)
            cv2.imwrite('test/gt.png',gt)
            pytorch_list.append(iou_pytorch)

    print 'pytorch',iter, np.sum(np.asarray(pytorch_list))/len(pytorch_list)
<|endoftext|>


<commit_before>import operator, json
import uuid
from functools import reduce

from django import forms
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db.models import Case, Count, Q, When
from django.db.models.functions import Lower
from django.http import HttpResponseRedirect, Http404

from django.shortcuts import reverse
from django.utils import timezone
from django.views import generic
from revproxy.views import ProxyView

from accounts.models import User
from accounts.utils import (get_permitted_triggers, status_tooltip_text,
                            update_trigger)
from guardian.mixins import LoginRequiredMixin
from guardian.shortcuts import (get_objects_for_user, get_perms,
                                get_users_with_perms)
from studies.forms import StudyEditForm, StudyForm, StudyBuildForm
from studies.models import Study, StudyLog
from project import settings

class StudyBuildView(PermissionRequiredMixin, generic.UpdateView):
    """
    StudyBuildView allows user to modify study structure - JSON field.
    """
    model = Study
    form_class = StudyBuildForm
    template_name = 'studies/study_json.html'
    permission_required = 'studies.can_edit_study'
    raise_exception = True

    def get_initial(self):
        """
        Returns the initial data to use for forms on this view.
        """
        initial = super().get_initial()
        structure = self.object.structure
        if structure:
            # Ensures that json displayed in edit form is valid json w/ double quotes,
            # so incorrect json is not saved back into the db
            initial['structure'] = json.dumps(structure)
        return initial

    def get_success_url(self):
        return reverse('exp:study-build', kwargs=dict(pk=self.object.id))

class StudyCreateView(LoginRequiredMixin, PermissionRequiredMixin, generic.CreateView):
    '''
    StudyCreateView allows a user to create a study and then redirects
    them to the detail view for that study.
    '''
    model = Study
    permission_required = 'studies.can_create_study'
    raise_exception = True
    form_class = StudyForm

    def form_valid(self, form):
        user = self.request.user
        form.instance.creator = user
        form.instance.organization = user.organization
        self.object = form.save()
        # TODO Should this be moved to the model - adding creator to study admin group on creation?
        self.add_creator_to_study_admin_group()
        return HttpResponseRedirect(self.get_success_url())

    def add_creator_to_study_admin_group(self):
        study_admin_group = self.object.study_admin_group
        study_admin_group.user_set.add(self.request.user)
        return study_admin_group

    def get_success_url(self):
        return reverse('exp:study-detail', kwargs=dict(pk=self.object.id))

class StudyListView(LoginRequiredMixin, generic.ListView):
    '''
    StudyListView shows a list of studies that a user has permission to.
    '''
    model = Study
    template_name = 'studies/study_list.html'

    def get_queryset(self, *args, **kwargs):
        request = self.request.GET
        queryset = get_objects_for_user(self.request.user, 'studies.can_view_study').exclude(state='archived')
        queryset = queryset.select_related('creator')
        queryset = queryset.annotate(completed_responses_count=Count(Case(When(responses__completed=True, then=1))))
        queryset = queryset.annotate(incomplete_responses_count=Count(Case(When(responses__completed=False, then=1))))

        state = request.get('state')
        if state and state != 'all':
            if state == 'myStudies':
                queryset = queryset.filter(creator=self.request.user)
            else:
                queryset = queryset.filter(state=state)

        match = request.get('match')
        if match:
            queryset = queryset.filter(reduce(operator.or_,
              (Q(name__icontains=term) | Q(short_description__icontains=term) for term in match.split())))

        sort = request.get('sort', '')
        if 'name' in sort:
            queryset = queryset.order_by(Lower('name').desc() if '-' in sort else Lower('name').asc())
        elif 'beginDate' in sort:
            # TODO optimize using subquery
            queryset = sorted(queryset, key=lambda t: t.begin_date or timezone.now(), reverse=True if '-' in sort else False)
        elif 'endDate' in sort:
            # TODO optimize using subquery
            queryset = sorted(queryset, key=lambda t: t.end_date or timezone.now(), reverse=True if '-' in sort else False)
        else:
            queryset = queryset.order_by(Lower('name'))

        return queryset

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['state'] = self.request.GET.get('state', 'all')
        context['match'] = self.request.GET.get('match') or ''
        context['sort'] = self.request.GET.get('sort') or ''
        return context

class StudyDetailView(LoginRequiredMixin, PermissionRequiredMixin, generic.DetailView):
    '''
    StudyDetailView shows information about a study.
    '''
    template_name = 'studies/study_detail.html'
    model = Study
    permission_required = 'studies.can_view_study'
    raise_exception = True

    def post(self, *args, **kwargs):
        update_trigger(self)
        if self.request.POST.get('clone_study'):
            clone = self.get_object().clone()
            clone.creator = self.request.user
            clone.organization = self.request.user.organization
            clone.save()
            return HttpResponseRedirect(reverse('exp:study-detail', kwargs=dict(pk=clone.pk)))
        return HttpResponseRedirect(reverse('exp:study-detail', kwargs=dict(pk=self.get_object().pk)))

    def get_queryset(self):
        queryset = super().get_queryset()
        queryset = queryset.annotate(completed_responses_count=Count(Case(When(responses__completed=True, then=1))))
        queryset = queryset.annotate(incomplete_responses_count=Count(Case(When(responses__completed=False, then=1))))
        return queryset

    @property
    def study_logs(self):
        ''' Returns a page object with 10 study logs'''
        logs_list = self.object.logs.all().order_by('-created_at')
        paginator = Paginator(logs_list, 10)
        page = self.request.GET.get('page')
        try:
            logs = paginator.page(page)
        except PageNotAnInteger:
            logs = paginator.page(1)
        except EmptyPage:
            logs = paginator.page(paginator.num_pages)
        return logs

    def get_context_data(self, **kwargs):
        context = super(StudyDetailView, self).get_context_data(**kwargs)
        context['triggers'] = get_permitted_triggers(self,
            self.object.machine.get_triggers(self.object.state))
        context['logs'] = self.study_logs
        state = self.object.state
        context['status_tooltip'] = status_tooltip_text.get(state, state)
        return context

class StudyUpdateView(LoginRequiredMixin, PermissionRequiredMixin, generic.UpdateView):
    '''
    StudyUpdateView allows user to edit study.
    '''
    template_name = 'studies/study_edit.html'
    form_class = StudyEditForm
    model = Study
    permission_required = 'studies.can_edit_study'
    raise_exception = True

    def get_study_researchers(self):
        '''  Pulls researchers that belong to Study Admin and Study Read groups '''
        study = self.get_object()
        return User.objects.filter(Q(groups__name=self.get_object().study_admin_group.name) | Q(groups__name=self.get_object().study_read_group.name)).distinct()

    def search_researchers(self):
        ''' Searches user first, last, and middle names for search query. Does not display researchers that are already on project '''
        search_query = self.request.GET.get('match', None)
        researchers_result = None
        if search_query:
            current_researcher_ids = self.get_study_researchers().values_list('id', flat=True)
            user_queryset = User.objects.filter(organization=self.request.user.organization,is_active=True)
            researchers_result = user_queryset.filter(reduce(operator.or_,
              (Q(family_name__icontains=term) | Q(given_name__icontains=term)  | Q(middle_name__icontains=term) for term in search_query.split()))).exclude(id__in=current_researcher_ids).distinct().order_by(Lower('family_name').asc())
            researchers_result = self.build_researchers_paginator(researchers_result)
        return researchers_result

    def build_researchers_paginator(self, researchers_result):
        '''
        Builds paginated search results for researchers
        '''
        paginator = Paginator(researchers_result, 5)
        page = self.request.GET.get('page')
        try:
            users = paginator.page(page)
        except PageNotAnInteger:
            users = paginator.page(1)
        except EmptyPage:
            users = paginator.page(paginator.num_pages)
        return users

    def manage_researcher_permissions(self):
        '''
        Handles adding, updating, and deleting researcher from study. Users are
        added to study read group by default.
        '''
        study_read_group = self.get_object().study_read_group
        study_admin_group = self.get_object().study_admin_group
        add_user = self.request.POST.get('add_user')
        remove_user = self.request.POST.get('remove_user')
        update_user = None
        if self.request.POST.get('name') == 'update_user':
             update_user = self.request.POST.get('pk')
             permissions = self.request.POST.get('value')

        if add_user:
            study_read_group.user_set.add(User.objects.get(pk=add_user))
        if remove_user:
            remove = User.objects.get(pk=remove_user)
            study_read_group.user_set.remove(remove)
            study_admin_group.user_set.remove(remove)
        if update_user:
            update = User.objects.get(pk=update_user)
            if permissions == 'study_admin':
                study_read_group.user_set.remove(update)
                study_admin_group.user_set.add(update)
            if permissions == 'study_read':
                study_read_group.user_set.add(update)
                study_admin_group.user_set.remove(update)

    def post(self, *args, **kwargs):
        '''
        Handles all post forms on page - 1) study metadata like name, short_description, etc. 2) researcher add 3) researcher update
        4) researcher delete 5) Changing study status / adding rejection comments
        '''
        if 'short_description' in self.request.POST:
            # Study metadata is being edited
            super().post(*args, **kwargs)

        update_trigger(self)
        self.manage_researcher_permissions()
        return HttpResponseRedirect(reverse('exp:study-edit', kwargs=dict(pk=self.get_object().pk)))

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        state = self.object.state

        context['current_researchers'] = self.get_study_researchers()
        context['users_result'] = self.search_researchers()
        context['search_query'] = self.request.GET.get('match')

        context['status_tooltip'] = status_tooltip_text.get(state, state)
        context['triggers'] = get_permitted_triggers(self, self.object.machine.get_triggers(state))
        return context

    def get_success_url(self):
        return reverse('exp:study-detail', kwargs={'pk': self.object.id})

class StudyResponsesList(LoginRequiredMixin, PermissionRequiredMixin, generic.DetailView):
    template_name = 'studies/study_responses.html'
    model = Study
    permission_required = 'studies.can_view_study_responses'
    raise_exception = True

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        orderby = self.request.GET.get('sort', None)
        study = context['study']
        context['responses'] = study.responses.all().order_by(orderby) if orderby else study.responses.all()
        context['response_data'] = [json.dumps({
            'sequence': resp.sequence,
            'conditions': resp.conditions,
            'exp_data': resp.exp_data,
            'participant_id': resp.child.user.id,
            'global_event_timings': resp.global_event_timings,
            'child_id': resp.child.id,
            'completed': resp.completed,
            'study_id': resp.study.id,
            'response_id': resp.id,
            }, indent=4) for resp in context['responses']]
        context['all_responses'] = ', '.join(context['response_data'])
        return context

class PreviewProxyView(ProxyView, LoginRequiredMixin):
    upstream = settings.EXPERIMENT_BASE_URL

    def dispatch(self, request, path, *args, **kwargs):
        return super().dispatch(request, path)
<commit_msg>Add comments to StudyListView methods.<commit_after>import operator, json
import uuid
from functools import reduce

from django import forms
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db.models import Case, Count, Q, When
from django.db.models.functions import Lower
from django.http import HttpResponseRedirect, Http404

from django.shortcuts import reverse
from django.utils import timezone
from django.views import generic
from revproxy.views import ProxyView

from accounts.models import User
from accounts.utils import (get_permitted_triggers, status_tooltip_text,
                            update_trigger)
from guardian.mixins import LoginRequiredMixin
from guardian.shortcuts import (get_objects_for_user, get_perms,
                                get_users_with_perms)
from studies.forms import StudyEditForm, StudyForm, StudyBuildForm
from studies.models import Study, StudyLog
from project import settings

class StudyBuildView(PermissionRequiredMixin, generic.UpdateView):
    """
    StudyBuildView allows user to modify study structure - JSON field.
    """
    model = Study
    form_class = StudyBuildForm
    template_name = 'studies/study_json.html'
    permission_required = 'studies.can_edit_study'
    raise_exception = True

    def get_initial(self):
        """
        Returns the initial data to use for forms on this view.
        """
        initial = super().get_initial()
        structure = self.object.structure
        if structure:
            # Ensures that json displayed in edit form is valid json w/ double quotes,
            # so incorrect json is not saved back into the db
            initial['structure'] = json.dumps(structure)
        return initial

    def get_success_url(self):
        return reverse('exp:study-build', kwargs=dict(pk=self.object.id))

class StudyCreateView(LoginRequiredMixin, PermissionRequiredMixin, generic.CreateView):
    '''
    StudyCreateView allows a user to create a study and then redirects
    them to the detail view for that study.
    '''
    model = Study
    permission_required = 'studies.can_create_study'
    raise_exception = True
    form_class = StudyForm

    def form_valid(self, form):
        user = self.request.user
        form.instance.creator = user
        form.instance.organization = user.organization
        self.object = form.save()
        # TODO Should this be moved to the model - adding creator to study admin group on creation?
        self.add_creator_to_study_admin_group()
        return HttpResponseRedirect(self.get_success_url())

    def add_creator_to_study_admin_group(self):
        study_admin_group = self.object.study_admin_group
        study_admin_group.user_set.add(self.request.user)
        return study_admin_group

    def get_success_url(self):
        return reverse('exp:study-detail', kwargs=dict(pk=self.object.id))

class StudyListView(LoginRequiredMixin, generic.ListView):
    '''
    StudyListView shows a list of studies that a user has permission to.
    '''
    model = Study
    template_name = 'studies/study_list.html'

    def get_queryset(self, *args, **kwargs):
        """
        Returns the list of items for the StudyListView - handles filtering on state, match,
        and sort.
        """
        request = self.request.GET
        queryset = get_objects_for_user(self.request.user, 'studies.can_view_study').exclude(state='archived')
        queryset = queryset.select_related('creator')
        queryset = queryset.annotate(completed_responses_count=Count(Case(When(responses__completed=True, then=1))))
        queryset = queryset.annotate(incomplete_responses_count=Count(Case(When(responses__completed=False, then=1))))

        state = request.get('state')
        if state and state != 'all':
            if state == 'myStudies':
                queryset = queryset.filter(creator=self.request.user)
            else:
                queryset = queryset.filter(state=state)

        match = request.get('match')
        if match:
            queryset = queryset.filter(reduce(operator.or_,
              (Q(name__icontains=term) | Q(short_description__icontains=term) for term in match.split())))

        sort = request.get('sort', '')
        if 'name' in sort:
            queryset = queryset.order_by(Lower('name').desc() if '-' in sort else Lower('name').asc())
        elif 'beginDate' in sort:
            # TODO optimize using subquery
            queryset = sorted(queryset, key=lambda t: t.begin_date or timezone.now(), reverse=True if '-' in sort else False)
        elif 'endDate' in sort:
            # TODO optimize using subquery
            queryset = sorted(queryset, key=lambda t: t.end_date or timezone.now(), reverse=True if '-' in sort else False)
        else:
            queryset = queryset.order_by(Lower('name'))

        return queryset

    def get_context_data(self, **kwargs):
        """
        Gets the context for the StudyListView and supplements with the state, match, and sort query params.
        """
        context = super().get_context_data(**kwargs)
        context['state'] = self.request.GET.get('state', 'all')
        context['match'] = self.request.GET.get('match', '')
        context['sort'] = self.request.GET.get('sort', 'name')
        return context

class StudyDetailView(LoginRequiredMixin, PermissionRequiredMixin, generic.DetailView):
    '''
    StudyDetailView shows information about a study.
    '''
    template_name = 'studies/study_detail.html'
    model = Study
    permission_required = 'studies.can_view_study'
    raise_exception = True

    def post(self, *args, **kwargs):
        update_trigger(self)
        if self.request.POST.get('clone_study'):
            clone = self.get_object().clone()
            clone.creator = self.request.user
            clone.organization = self.request.user.organization
            clone.save()
            return HttpResponseRedirect(reverse('exp:study-detail', kwargs=dict(pk=clone.pk)))
        return HttpResponseRedirect(reverse('exp:study-detail', kwargs=dict(pk=self.get_object().pk)))

    def get_queryset(self):
        queryset = super().get_queryset()
        queryset = queryset.annotate(completed_responses_count=Count(Case(When(responses__completed=True, then=1))))
        queryset = queryset.annotate(incomplete_responses_count=Count(Case(When(responses__completed=False, then=1))))
        return queryset

    @property
    def study_logs(self):
        ''' Returns a page object with 10 study logs'''
        logs_list = self.object.logs.all().order_by('-created_at')
        paginator = Paginator(logs_list, 10)
        page = self.request.GET.get('page')
        try:
            logs = paginator.page(page)
        except PageNotAnInteger:
            logs = paginator.page(1)
        except EmptyPage:
            logs = paginator.page(paginator.num_pages)
        return logs

    def get_context_data(self, **kwargs):
        context = super(StudyDetailView, self).get_context_data(**kwargs)
        context['triggers'] = get_permitted_triggers(self,
            self.object.machine.get_triggers(self.object.state))
        context['logs'] = self.study_logs
        state = self.object.state
        context['status_tooltip'] = status_tooltip_text.get(state, state)
        return context

class StudyUpdateView(LoginRequiredMixin, PermissionRequiredMixin, generic.UpdateView):
    '''
    StudyUpdateView allows user to edit study.
    '''
    template_name = 'studies/study_edit.html'
    form_class = StudyEditForm
    model = Study
    permission_required = 'studies.can_edit_study'
    raise_exception = True

    def get_study_researchers(self):
        '''  Pulls researchers that belong to Study Admin and Study Read groups '''
        study = self.get_object()
        return User.objects.filter(Q(groups__name=self.get_object().study_admin_group.name) | Q(groups__name=self.get_object().study_read_group.name)).distinct()

    def search_researchers(self):
        ''' Searches user first, last, and middle names for search query. Does not display researchers that are already on project '''
        search_query = self.request.GET.get('match', None)
        researchers_result = None
        if search_query:
            current_researcher_ids = self.get_study_researchers().values_list('id', flat=True)
            user_queryset = User.objects.filter(organization=self.request.user.organization,is_active=True)
            researchers_result = user_queryset.filter(reduce(operator.or_,
              (Q(family_name__icontains=term) | Q(given_name__icontains=term)  | Q(middle_name__icontains=term) for term in search_query.split()))).exclude(id__in=current_researcher_ids).distinct().order_by(Lower('family_name').asc())
            researchers_result = self.build_researchers_paginator(researchers_result)
        return researchers_result

    def build_researchers_paginator(self, researchers_result):
        '''
        Builds paginated search results for researchers
        '''
        paginator = Paginator(researchers_result, 5)
        page = self.request.GET.get('page')
        try:
            users = paginator.page(page)
        except PageNotAnInteger:
            users = paginator.page(1)
        except EmptyPage:
            users = paginator.page(paginator.num_pages)
        return users

    def manage_researcher_permissions(self):
        '''
        Handles adding, updating, and deleting researcher from study. Users are
        added to study read group by default.
        '''
        study_read_group = self.get_object().study_read_group
        study_admin_group = self.get_object().study_admin_group
        add_user = self.request.POST.get('add_user')
        remove_user = self.request.POST.get('remove_user')
        update_user = None
        if self.request.POST.get('name') == 'update_user':
             update_user = self.request.POST.get('pk')
             permissions = self.request.POST.get('value')

        if add_user:
            study_read_group.user_set.add(User.objects.get(pk=add_user))
        if remove_user:
            remove = User.objects.get(pk=remove_user)
            study_read_group.user_set.remove(remove)
            study_admin_group.user_set.remove(remove)
        if update_user:
            update = User.objects.get(pk=update_user)
            if permissions == 'study_admin':
                study_read_group.user_set.remove(update)
                study_admin_group.user_set.add(update)
            if permissions == 'study_read':
                study_read_group.user_set.add(update)
                study_admin_group.user_set.remove(update)

    def post(self, *args, **kwargs):
        '''
        Handles all post forms on page - 1) study metadata like name, short_description, etc. 2) researcher add 3) researcher update
        4) researcher delete 5) Changing study status / adding rejection comments
        '''
        if 'short_description' in self.request.POST:
            # Study metadata is being edited
            super().post(*args, **kwargs)

        update_trigger(self)
        self.manage_researcher_permissions()
        return HttpResponseRedirect(reverse('exp:study-edit', kwargs=dict(pk=self.get_object().pk)))

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        state = self.object.state

        context['current_researchers'] = self.get_study_researchers()
        context['users_result'] = self.search_researchers()
        context['search_query'] = self.request.GET.get('match')

        context['status_tooltip'] = status_tooltip_text.get(state, state)
        context['triggers'] = get_permitted_triggers(self, self.object.machine.get_triggers(state))
        return context

    def get_success_url(self):
        return reverse('exp:study-detail', kwargs={'pk': self.object.id})

class StudyResponsesList(LoginRequiredMixin, PermissionRequiredMixin, generic.DetailView):
    template_name = 'studies/study_responses.html'
    model = Study
    permission_required = 'studies.can_view_study_responses'
    raise_exception = True

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        orderby = self.request.GET.get('sort', None)
        study = context['study']
        context['responses'] = study.responses.all().order_by(orderby) if orderby else study.responses.all()
        context['response_data'] = [json.dumps({
            'sequence': resp.sequence,
            'conditions': resp.conditions,
            'exp_data': resp.exp_data,
            'participant_id': resp.child.user.id,
            'global_event_timings': resp.global_event_timings,
            'child_id': resp.child.id,
            'completed': resp.completed,
            'study_id': resp.study.id,
            'response_id': resp.id,
            }, indent=4) for resp in context['responses']]
        context['all_responses'] = ', '.join(context['response_data'])
        return context

class PreviewProxyView(ProxyView, LoginRequiredMixin):
    upstream = settings.EXPERIMENT_BASE_URL

    def dispatch(self, request, path, *args, **kwargs):
        return super().dispatch(request, path)
<|endoftext|>


<commit_before>__import__('pkg_resources').declare_namespace(__name__)
<commit_msg>moved to pkgutil for the namespace package<commit_after>from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# The MIT License (MIT)
#
# Copyright (c) 2015 Taio Jia (jiasir) <jiasir@icloud.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__author__ = 'jiasir'

import platform
import os

import playback
import argparse
import shutil
from fabric.api import *

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()

group.add_argument('-a', '--ansible', help='use ansible to provisioning')
parser.add_argument('-v', '--vars', help='vars to playbooks')
group.add_argument('-p', '--puppet', help='use puppet to provisioning')
group.add_argument('-s', '--saltstack', help='use saltstack to provisioning')
group.add_argument('-c', '--chef', help='use chef to provisioning')
group.add_argument('-j', '--juju', help='use juju to provisioning')
group.add_argument('-d', '--deploy', help='do deploy directly', action='store_true', default=False)
group.add_argument('-i', '--init', help='initialize the configuration file', action='store_true', default=False)
parser.add_argument('-r', '--roles', help='which roles to deploy', type=str, choices=['haproxy'])
group.add_argument('--novadocker', help='use docker libvirt for compute', action='store_true', default=False)
parser.add_argument('--user', help='which user to login remote server', action='store', type=str)
parser.add_argument('--hosts', help='hosts to deploy', action='store', type=str)
group.add_argument('--redis', help='deploy redis', action='store_true', default=False)

args = parser.parse_args()

def haproxy_deploy():
    haproxy = playback.haproxy.Haproxy()
    [haproxy.deploy() for i in haproxy.host_string()]

def init():
    if os.path.isdir('.playback'):
        shutil.rmtree('.playback')
    elif platform.system() == 'Darwin':
        shutil.copytree('/Library/Python/2.7/site-packages/playback/config', '.playback')
    elif platform.system() == 'Linux':
        shutil.copytree('/usr/local/lib/python2.7/dist-packages/playback/config', '.playback')

def ansible(playbook, vars):
    command = 'ansible-playbook {playbook} --extra-vars "{vars}" -vvvv'.format(playbook=playbook, vars=vars)
    os.system(command)

# TODO provider abstractly

def puppet():
    pass

def chef():
    pass

def saltstack():
    pass

def juju():
    pass

def deploy_docker(user, hosts):
    docker = playback.nova_docker.Docker(user, hosts)
    execute(docker.docker_prerequisites)
    execute(docker.docker_install)
    execute(docker.docker_group)
    execute(docker.docker_driver)
    execute(docker.nova_config)
    execute(docker.glance_config, hosts=['controller01', 'controller02'])

def define_user():
    """
    :return: user string
    """
    return args.user

def define_hosts():
    """
    :return: hosts list
    """
    return args.hosts.split(',')

def cmd():
    if args.init:
        init()

    if args.ansible:
        ansible(args.ansible, args.vars)

    if args.puppet:
        puppet()

    if args.chef:
        chef()

    if args.saltstack:
        saltstack()

    if args.juju:
        juju()

    if args.roles == 'haproxy' and args.deploy:
        haproxy_deploy()

    if args.novadocker:
        user = args.user
        hosts = args.hosts.split(',')
        deploy_docker(user, hosts)

    if args.redis:
        user = define_user()
        hosts = define_hosts()
        redis = playback.redis.Redis(user, hosts)
        execute(redis.install_redis)<commit_msg>Add dest for argparse<commit_after>#!/usr/bin/env python

# The MIT License (MIT)
#
# Copyright (c) 2015 Taio Jia (jiasir) <jiasir@icloud.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__author__ = 'jiasir'

import platform
import os

import playback
import argparse
import shutil
from fabric.api import *

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()

group.add_argument('-a', '--ansible', help='use ansible to provisioning')
parser.add_argument('-v', '--vars', help='vars to playbooks')
group.add_argument('-p', '--puppet', help='use puppet to provisioning')
group.add_argument('-s', '--saltstack', help='use saltstack to provisioning')
group.add_argument('-c', '--chef', help='use chef to provisioning')
group.add_argument('-j', '--juju', help='use juju to provisioning')
group.add_argument('-d', '--deploy', help='do deploy directly', action='store_true', default=False)
group.add_argument('-i', '--init', help='initialize the configuration file', action='store_true', default=False)
parser.add_argument('-r', '--roles', help='which roles to deploy', type=str, choices=['haproxy'])
group.add_argument('--novadocker', help='use docker libvirt for compute', action='store_true', default=False)
parser.add_argument('--user', help='which user to login remote server', action='store', type=str)
parser.add_argument('--hosts', help='hosts to deploy', action='store', dest='hosts')
group.add_argument('--redis', help='deploy redis', action='store_true', default=False)

args = parser.parse_args()

def haproxy_deploy():
    haproxy = playback.haproxy.Haproxy()
    [haproxy.deploy() for i in haproxy.host_string()]

def init():
    if os.path.isdir('.playback'):
        shutil.rmtree('.playback')
    elif platform.system() == 'Darwin':
        shutil.copytree('/Library/Python/2.7/site-packages/playback/config', '.playback')
    elif platform.system() == 'Linux':
        shutil.copytree('/usr/local/lib/python2.7/dist-packages/playback/config', '.playback')

def ansible(playbook, vars):
    command = 'ansible-playbook {playbook} --extra-vars "{vars}" -vvvv'.format(playbook=playbook, vars=vars)
    os.system(command)

# TODO provider abstractly

def puppet():
    pass

def chef():
    pass

def saltstack():
    pass

def juju():
    pass

def deploy_docker(user, hosts):
    docker = playback.nova_docker.Docker(user, hosts)
    execute(docker.docker_prerequisites)
    execute(docker.docker_install)
    execute(docker.docker_group)
    execute(docker.docker_driver)
    execute(docker.nova_config)
    execute(docker.glance_config, hosts=['controller01', 'controller02'])

def define_user():
    """
    :return: user string
    """
    return args.user

def define_hosts():
    """
    :return: hosts list
    """
    return args.hosts.split(',')

def cmd():
    if args.init:
        init()

    if args.ansible:
        ansible(args.ansible, args.vars)

    if args.puppet:
        puppet()

    if args.chef:
        chef()

    if args.saltstack:
        saltstack()

    if args.juju:
        juju()

    if args.roles == 'haproxy' and args.deploy:
        haproxy_deploy()

    if args.novadocker:
        user = args.user
        hosts = args.hosts.split(',')
        deploy_docker(user, hosts)

    if args.redis:
        user = define_user()
        hosts = define_hosts()
        redis = playback.redis.Redis(user, hosts)
        execute(redis.install_redis)
<|endoftext|>