<commit_before>import datetime
import unicodedata

from django.core.urlresolvers import reverse
from django.core.mail import (EmailMultiAlternatives, EmailMessage,
                              send_mass_mail)
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.db.models.signals import post_save
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils import translation
from django.utils import simplejson as json
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404

from userena.models import UserenaLanguageBaseProfile
from userena.utils import get_gravatar
from userena import settings as userena_settings
from guardian.shortcuts import *

from actstream.actions import follow, is_following
from actstream.signals import action

from agora_site.misc.utils import (JSONField, geolocate_ip, send_action,
                                   get_base_email_context)
from agora import Agora
from election import Election
from castvote import CastVote
from authority import Authority
from delegateelectioncount import DelegateElectionCount

class Profile(UserenaLanguageBaseProfile):
    '''
    Profile used together with django User class, and accessible via
    user.get_profile(), because  in settings we have configured:

    AUTH_PROFILE_MODULE = 'agora_site.agora_core.models.Profile'

    See https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-AUTH_PROFILE_MODULE
    for more details.
    '''
    user = models.OneToOneField(User)

    class Meta:
        app_label = 'agora_core'

    def get_fullname(self):
        '''
        Returns the full user name
        '''
        if self.user.last_name:
            return self.user.first_name + ' ' + self.user.last_name
        else:
            return self.user.first_name

    def get_initials(self):
        '''
        Returns the initials of the profile
        '''
        base_text = self.get_fullname().strip()
        initials = u""
        for word in base_text.split(" "):
            word = word.strip()
            if not word:
                continue
            initials += word[0].upper()
        return unicodedata.normalize('NFKD', initials).encode('ascii','ignore')

    def delete_mugshot(self):
        if self.mugshot.name in ['gravatar', 'initials', '']:
            self.mugshot.name = ""
            return
        self.mugshot.delete()

    def get_initials_mugshot(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE):
        if userena_settings.USERENA_MUGSHOT_DEFAULT == 'blank-unitials-ssl':
            return 'https://unitials.com/mugshot/%s/%s.png' % (
                custom_size, self.get_initials()
            )
        elif userena_settings.USERENA_MUGSHOT_DEFAULT == 'blank-unitials':
            return 'http://unitials.com/mugshot/%s/%s.png' % (
                custom_size, self.get_initials()
            )

    def get_gravatar_mugshot(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE, force_default=False):
        d = self.get_initials_mugshot(custom_size)
        return get_gravatar(self.user.email, custom_size, d, force_default)

    def get_big_mugshot(self):
        return self.get_mugshot_url(170)

    def add_to_agora(self, request=None, agora_name=None, agora_id=None, silent=True):
        '''
        Add the user to the specified agora. The agora is specified by its full
        name or id, for example agora_name="username/agoraname" or agora_id=3.
        '''

        if agora_name:
            username, agoraname = agora_name.split("/")
            agora = get_object_or_404(Agora, name=agoraname,
                creator__username=username)
        else:
            agora = get_object_or_404(Agora, pk=agora_id)

        if agora.members.filter(pk=self.user.id).exists():
            return
        agora.members.add(self.user)
        agora.save()

        if not silent:
            send_action(self.user, verb='joined', action_object=agora, request=request)

        remove_perm('requested_membership', self.user, agora)
        remove_perm('denied_requested_membership', self.user, agora)

        if not is_following(self.user, agora) and not silent:
            follow(self.user, agora, actor_only=False, request=request)

        # Mail to the user
        if not self.has_perms('receive_email_updates') or silent:
            return

        translation.activate(self.user.get_profile().lang_code)
        context = get_base_email_context(request)
        context.update(dict(
            agora=agora,
            other_user=self.user,
            notification_text=_('You just joined %(agora)s. '
                'Congratulations!') % dict(agora=agora.get_full_name()),
            to=self.user
        ))

        email = EmailMultiAlternatives(
            subject=_('%(site)s - you are now member of %(agora)s') % dict(
                        site=Site.objects.get_current().domain,
                        agora=agora.get_full_name()
                    ),
            body=render_to_string('agora_core/emails/agora_notification.txt',
                context),
            to=[self.user.email])

        email.attach_alternative(
            render_to_string('agora_core/emails/agora_notification.html',
                context), "text/html")
        email.send()
        translation.deactivate()

    def get_mugshot_url(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE, force_default=False):
        """
        Returns the image containing the mugshot for the user.

        The mugshot can be a uploaded image or a Gravatar.

        Gravatar functionality will only be used when
        ``USERENA_MUGSHOT_GRAVATAR`` is set to ``True``.

        :return:
            ``None`` when Gravatar is not used and no default image is supplied
            by ``USERENA_MUGSHOT_DEFAULT``.

        """
        # First check for a mugshot and if any return that.
        if self.mugshot.name == "gravatar":
            return self.get_gravatar_mugshot(custom_size, force_default=force_default)
        elif self.mugshot.name == "initials":
            return self.get_initials_mugshot(custom_size)
        elif self.mugshot:
            return settings.MEDIA_URL +\
                   settings.MUGSHOTS_DIR +\
                   self.mugshot.name.split("/")[-1]

        # Use Gravatar if the user wants to.
        if userena_settings.USERENA_MUGSHOT_GRAVATAR:
            return self.get_gravatar_mugshot(custom_size, force_default=force_default)

        # Gravatar not used, check for a default image.
        else:
            if userena_settings.USERENA_MUGSHOT_DEFAULT not in ['404', 'mm',
                'identicon', 'monsterid', 'wavatar', 'blank']:
                return userena_settings.USERENA_MUGSHOT_DEFAULT
            else:
                return None

    def get_short_description(self):
        '''
        Returns a short description of the user
        '''
        if self.short_description:
            return self.short_description
        else:
            return _('Is a member of %(num_agoras)d agoras and has emitted '
                ' %(num_votes)d direct votes.') % dict(
                    num_agoras=self.user.agoras.count(),
                    num_votes=self.count_direct_votes())

    def get_first_name_or_nick(self):
        if self.user.first_name:
            return self.user.first_name
        else:
            return self.user.username

    def has_perms(self, permission_name, user=None):
        '''
        Return whether a given user has a given permission name
        '''
        isanon = user and user.is_anonymous()

        if permission_name == 'receive_email_updates':
            from django.core.validators import validate_email
            from django.core.exceptions import ValidationError
            try:
                validate_email(self.user.email)
            except ValidationError:
                return False
            return self.email_updates
        elif permission_name == 'receive_mail':
            from django.core.validators import validate_email
            from django.core.exceptions import ValidationError
            if user.is_anonymous():
                return False
            # HACK TODO a big hack yes it is
            if settings.AGORA_CREATION_PERMISSIONS == "superusers-only" and\
                    user.administrated_agoras.count() > 0:
                return True
            # only admins of the agora the user is in can send the user an email
            if not user.administrated_agoras.only('id').filter(
                    id__in=self.user.agoras.only('id').all().query).exists():
                return False
            try:
                validate_email(self.user.email)
            except ValidationError:
                return False
            return not isanon
        else:
            return False

    def get_perms(self, user):
        '''
        Returns a list of permissions for a given user calling to self.has_perms()
        '''
        return [perm for perm in ('receive_email_updates', 'receive_mail')
                if self.has_perms(perm, user)]

    short_description = models.CharField(_('Short Description'), max_length=140, null=True, blank=True)

    biography = models.TextField(_('Biography'), null=True, blank=True, default='')

    # This marks the date of the last activity item known to be read by the user
    # so that later on we can for example send to the user update email only
    # showing activity from this date on
    last_activity_read_date = models.DateTimeField(_(u'Last Activity Read Date'), auto_now_add=True, editable=True)

    # Saving the user language allows sending emails to him in his desired
    # language (among other things)
    lang_code = models.CharField(_("Language Code"), max_length=10, default='', null=True, blank=True)

    email_updates = models.BooleanField(_("Receive email updates"),
        default=True)

    # Stores extra data
    extra = JSONField(_('Extra'), null=True, blank=True, default='')

    def get_open_elections(self, searchquery = None):
        '''
        Returns the list of current and future elections that will or are
        taking place in our agoras.
        '''
        elections = Election.objects.filter(
            Q(voting_extended_until_date__gt=timezone.now()) |
            Q(voting_extended_until_date=None, voting_starts_at_date__lt=timezone.now()),
            Q(is_approved=True, agora__in=self.user.agoras.all())).filter(archived_at_date=None)

        if searchquery and len(searchquery) > 1:
            elections = elections.filter(pretty_name__icontains=searchquery)

        return elections.order_by('-voting_extended_until_date',
                '-voting_starts_at_date')

    def get_requested_elections(self):
        '''
        Returns the list of requested elections related to us.
        '''
        return Election.objects.filter(
            Q(agora__in=self.user.administrated_agoras.all()) | Q(creator=self.user),
            Q(is_approved=False) | Q(result_tallied_at_date=None)
        ).filter(archived_at_date=None).exclude(name='delegation').order_by('-voting_extended_until_date', '-voting_starts_at_date')

    def count_direct_votes(self):
        '''
        Returns the list of valid direct votes by this user
        '''
        return CastVote.objects.filter(voter=self.user, is_direct=True, is_counted=True).count()

    def get_participated_elections(self):
        '''
        Returns the list of elections in which the user participated, either
        via a direct or a delegated vote
        '''
        user_direct_votes=CastVote.objects.filter(voter=self.user, is_direct=True, is_counted=True).all()
        user_delegated_votes=CastVote.objects.filter(voter=self.user).all()
        return Election.objects.filter(agora__isnull=False,
            result_tallied_at_date__isnull=False).filter(
                Q(delegated_votes__in=user_delegated_votes) |
                Q(cast_votes__in=user_direct_votes)).distinct().order_by('-result_tallied_at_date','-voting_extended_until_date')

    def has_delegated_in_agora(self, agora):
        '''
        Returns whether this user has currently delegated his vote in a given
        agora.
        '''
        return bool(CastVote.objects.filter(voter=self.user, is_direct=False,
            election=agora.delegation_election, is_counted=True).count())

    def get_delegation_in_agora(self, agora):
        '''
        Returns this user current vote regarding his delegation (if any)
        '''
        try:
            return CastVote.objects.filter(voter=self.user, is_direct=False,
                election=agora.delegation_election, is_counted=True).order_by('-casted_at_date')[0]
        except Exception, e:
            return None

    def get_vote_in_election(self, election):
        '''
        Returns the vote of this user in the given agora if any. Note: if the
        vote is a delegated one, this only works for tallied elections.
        '''
        if election.cast_votes.filter(voter=self.user, is_counted=True).count() == 1:
            return election.cast_votes.filter(voter=self.user, is_counted=True)[0]
        else:
            votes = election.delegated_votes.filter(voter=self.user)
            if len(votes) == 0:
                return None

            return votes[0]

    def get_link(self):
        return reverse('user-view', kwargs=dict(username=self.user.username))

# definition of UserProfile from above
# ...

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        Profile.objects.create(user=instance)

post_save.connect(create_user_profile, sender=User)

from tastypie.models import create_api_key
post_save.connect(create_api_key, sender=User)

from django.db.models.signals import class_prepared
def modify_auth_user(sender, *args, **kwargs):
    # You can't just do `if sender == django.contrib.auth.models.User`
    # because you would have to import the model
    # You have to test using __name__ and __module__
    if sender.__name__ == "User" and sender.__module__ == "django.contrib.auth.models":
        sender._meta.get_field("first_name").max_length = 140
<commit_msg>fix silent option in profile.add_to_agora<commit_after>import datetime
import unicodedata

from django.core.urlresolvers import reverse
from django.core.mail import (EmailMultiAlternatives, EmailMessage,
                              send_mass_mail)
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.db.models.signals import post_save
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils import translation
from django.utils import simplejson as json
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404

from userena.models import UserenaLanguageBaseProfile
from userena.utils import get_gravatar
from userena import settings as userena_settings
from guardian.shortcuts import *

from actstream.actions import follow, is_following
from actstream.signals import action

from agora_site.misc.utils import (JSONField, geolocate_ip, send_action,
                                   get_base_email_context)
from agora import Agora
from election import Election
from castvote import CastVote
from authority import Authority
from delegateelectioncount import DelegateElectionCount

class Profile(UserenaLanguageBaseProfile):
    '''
    Profile used together with django User class, and accessible via
    user.get_profile(), because  in settings we have configured:

    AUTH_PROFILE_MODULE = 'agora_site.agora_core.models.Profile'

    See https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-AUTH_PROFILE_MODULE
    for more details.
    '''
    user = models.OneToOneField(User)

    class Meta:
        app_label = 'agora_core'

    def get_fullname(self):
        '''
        Returns the full user name
        '''
        if self.user.last_name:
            return self.user.first_name + ' ' + self.user.last_name
        else:
            return self.user.first_name

    def get_initials(self):
        '''
        Returns the initials of the profile
        '''
        base_text = self.get_fullname().strip()
        initials = u""
        for word in base_text.split(" "):
            word = word.strip()
            if not word:
                continue
            initials += word[0].upper()
        return unicodedata.normalize('NFKD', initials).encode('ascii','ignore')

    def delete_mugshot(self):
        if self.mugshot.name in ['gravatar', 'initials', '']:
            self.mugshot.name = ""
            return
        self.mugshot.delete()

    def get_initials_mugshot(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE):
        if userena_settings.USERENA_MUGSHOT_DEFAULT == 'blank-unitials-ssl':
            return 'https://unitials.com/mugshot/%s/%s.png' % (
                custom_size, self.get_initials()
            )
        elif userena_settings.USERENA_MUGSHOT_DEFAULT == 'blank-unitials':
            return 'http://unitials.com/mugshot/%s/%s.png' % (
                custom_size, self.get_initials()
            )

    def get_gravatar_mugshot(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE, force_default=False):
        d = self.get_initials_mugshot(custom_size)
        return get_gravatar(self.user.email, custom_size, d, force_default)

    def get_big_mugshot(self):
        return self.get_mugshot_url(170)

    def add_to_agora(self, request=None, agora_name=None, agora_id=None, silent=False):
        '''
        Add the user to the specified agora. The agora is specified by its full
        name or id, for example agora_name="username/agoraname" or agora_id=3.
        '''

        if agora_name:
            username, agoraname = agora_name.split("/")
            agora = get_object_or_404(Agora, name=agoraname,
                creator__username=username)
        else:
            agora = get_object_or_404(Agora, pk=agora_id)

        if agora.members.filter(pk=self.user.id).exists():
            return
        agora.members.add(self.user)
        agora.save()

        if not silent:
            send_action(self.user, verb='joined', action_object=agora, request=request)

        remove_perm('requested_membership', self.user, agora)
        remove_perm('denied_requested_membership', self.user, agora)

        if not silent and not not is_following(self.user, agora):
            follow(self.user, agora, actor_only=False, request=request)

        # Mail to the user
        if silent or not self.has_perms('receive_email_updates'):
            return

        translation.activate(self.user.get_profile().lang_code)
        context = get_base_email_context(request)
        context.update(dict(
            agora=agora,
            other_user=self.user,
            notification_text=_('You just joined %(agora)s. '
                'Congratulations!') % dict(agora=agora.get_full_name()),
            to=self.user
        ))

        email = EmailMultiAlternatives(
            subject=_('%(site)s - you are now member of %(agora)s') % dict(
                        site=Site.objects.get_current().domain,
                        agora=agora.get_full_name()
                    ),
            body=render_to_string('agora_core/emails/agora_notification.txt',
                context),
            to=[self.user.email])

        email.attach_alternative(
            render_to_string('agora_core/emails/agora_notification.html',
                context), "text/html")
        email.send()
        translation.deactivate()

    def get_mugshot_url(self, custom_size = userena_settings.USERENA_MUGSHOT_SIZE, force_default=False):
        """
        Returns the image containing the mugshot for the user.

        The mugshot can be a uploaded image or a Gravatar.

        Gravatar functionality will only be used when
        ``USERENA_MUGSHOT_GRAVATAR`` is set to ``True``.

        :return:
            ``None`` when Gravatar is not used and no default image is supplied
            by ``USERENA_MUGSHOT_DEFAULT``.

        """
        # First check for a mugshot and if any return that.
        if self.mugshot.name == "gravatar":
            return self.get_gravatar_mugshot(custom_size, force_default=force_default)
        elif self.mugshot.name == "initials":
            return self.get_initials_mugshot(custom_size)
        elif self.mugshot:
            return settings.MEDIA_URL +\
                   settings.MUGSHOTS_DIR +\
                   self.mugshot.name.split("/")[-1]

        # Use Gravatar if the user wants to.
        if userena_settings.USERENA_MUGSHOT_GRAVATAR:
            return self.get_gravatar_mugshot(custom_size, force_default=force_default)

        # Gravatar not used, check for a default image.
        else:
            if userena_settings.USERENA_MUGSHOT_DEFAULT not in ['404', 'mm',
                'identicon', 'monsterid', 'wavatar', 'blank']:
                return userena_settings.USERENA_MUGSHOT_DEFAULT
            else:
                return None

    def get_short_description(self):
        '''
        Returns a short description of the user
        '''
        if self.short_description:
            return self.short_description
        else:
            return _('Is a member of %(num_agoras)d agoras and has emitted '
                ' %(num_votes)d direct votes.') % dict(
                    num_agoras=self.user.agoras.count(),
                    num_votes=self.count_direct_votes())

    def get_first_name_or_nick(self):
        if self.user.first_name:
            return self.user.first_name
        else:
            return self.user.username

    def has_perms(self, permission_name, user=None):
        '''
        Return whether a given user has a given permission name
        '''
        isanon = user and user.is_anonymous()

        if permission_name == 'receive_email_updates':
            from django.core.validators import validate_email
            from django.core.exceptions import ValidationError
            try:
                validate_email(self.user.email)
            except ValidationError:
                return False
            return self.email_updates
        elif permission_name == 'receive_mail':
            from django.core.validators import validate_email
            from django.core.exceptions import ValidationError
            if user.is_anonymous():
                return False
            # HACK TODO a big hack yes it is
            if settings.AGORA_CREATION_PERMISSIONS == "superusers-only" and\
                    user.administrated_agoras.count() > 0:
                return True
            # only admins of the agora the user is in can send the user an email
            if not user.administrated_agoras.only('id').filter(
                    id__in=self.user.agoras.only('id').all().query).exists():
                return False
            try:
                validate_email(self.user.email)
            except ValidationError:
                return False
            return not isanon
        else:
            return False

    def get_perms(self, user):
        '''
        Returns a list of permissions for a given user calling to self.has_perms()
        '''
        return [perm for perm in ('receive_email_updates', 'receive_mail')
                if self.has_perms(perm, user)]

    short_description = models.CharField(_('Short Description'), max_length=140, null=True, blank=True)

    biography = models.TextField(_('Biography'), null=True, blank=True, default='')

    # This marks the date of the last activity item known to be read by the user
    # so that later on we can for example send to the user update email only
    # showing activity from this date on
    last_activity_read_date = models.DateTimeField(_(u'Last Activity Read Date'), auto_now_add=True, editable=True)

    # Saving the user language allows sending emails to him in his desired
    # language (among other things)
    lang_code = models.CharField(_("Language Code"), max_length=10, default='', null=True, blank=True)

    email_updates = models.BooleanField(_("Receive email updates"),
        default=True)

    # Stores extra data
    extra = JSONField(_('Extra'), null=True, blank=True, default='')

    def get_open_elections(self, searchquery = None):
        '''
        Returns the list of current and future elections that will or are
        taking place in our agoras.
        '''
        elections = Election.objects.filter(
            Q(voting_extended_until_date__gt=timezone.now()) |
            Q(voting_extended_until_date=None, voting_starts_at_date__lt=timezone.now()),
            Q(is_approved=True, agora__in=self.user.agoras.all())).filter(archived_at_date=None)

        if searchquery and len(searchquery) > 1:
            elections = elections.filter(pretty_name__icontains=searchquery)

        return elections.order_by('-voting_extended_until_date',
                '-voting_starts_at_date')

    def get_requested_elections(self):
        '''
        Returns the list of requested elections related to us.
        '''
        return Election.objects.filter(
            Q(agora__in=self.user.administrated_agoras.all()) | Q(creator=self.user),
            Q(is_approved=False) | Q(result_tallied_at_date=None)
        ).filter(archived_at_date=None).exclude(name='delegation').order_by('-voting_extended_until_date', '-voting_starts_at_date')

    def count_direct_votes(self):
        '''
        Returns the list of valid direct votes by this user
        '''
        return CastVote.objects.filter(voter=self.user, is_direct=True, is_counted=True).count()

    def get_participated_elections(self):
        '''
        Returns the list of elections in which the user participated, either
        via a direct or a delegated vote
        '''
        user_direct_votes=CastVote.objects.filter(voter=self.user, is_direct=True, is_counted=True).all()
        user_delegated_votes=CastVote.objects.filter(voter=self.user).all()
        return Election.objects.filter(agora__isnull=False,
            result_tallied_at_date__isnull=False).filter(
                Q(delegated_votes__in=user_delegated_votes) |
                Q(cast_votes__in=user_direct_votes)).distinct().order_by('-result_tallied_at_date','-voting_extended_until_date')

    def has_delegated_in_agora(self, agora):
        '''
        Returns whether this user has currently delegated his vote in a given
        agora.
        '''
        return bool(CastVote.objects.filter(voter=self.user, is_direct=False,
            election=agora.delegation_election, is_counted=True).count())

    def get_delegation_in_agora(self, agora):
        '''
        Returns this user current vote regarding his delegation (if any)
        '''
        try:
            return CastVote.objects.filter(voter=self.user, is_direct=False,
                election=agora.delegation_election, is_counted=True).order_by('-casted_at_date')[0]
        except Exception, e:
            return None

    def get_vote_in_election(self, election):
        '''
        Returns the vote of this user in the given agora if any. Note: if the
        vote is a delegated one, this only works for tallied elections.
        '''
        if election.cast_votes.filter(voter=self.user, is_counted=True).count() == 1:
            return election.cast_votes.filter(voter=self.user, is_counted=True)[0]
        else:
            votes = election.delegated_votes.filter(voter=self.user)
            if len(votes) == 0:
                return None

            return votes[0]

    def get_link(self):
        return reverse('user-view', kwargs=dict(username=self.user.username))

# definition of UserProfile from above
# ...

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        Profile.objects.create(user=instance)

post_save.connect(create_user_profile, sender=User)

from tastypie.models import create_api_key
post_save.connect(create_api_key, sender=User)

from django.db.models.signals import class_prepared
def modify_auth_user(sender, *args, **kwargs):
    # You can't just do `if sender == django.contrib.auth.models.User`
    # because you would have to import the model
    # You have to test using __name__ and __module__
    if sender.__name__ == "User" and sender.__module__ == "django.contrib.auth.models":
        sender._meta.get_field("first_name").max_length = 140
<|endoftext|>


<commit_before>"""
Class to manage the dbcollection.json cache file.
"""

import os
import errno
import json

class CacheManager:
    """ Class to manage the dbcollection cache data """

    def __init__(self):
        """
        Initialize class.
        """

        # setup cache paths
        self.setup_paths()

        # create cache file (if it does not exist)
        if not os.path.exists(self.cache_fname):
            self.create_cache_file_disk(self.cache_fname)

        # load cache data file
        self.data = self.read_data_cache()

    def setup_paths(self):
        """Setup the cache/data directories for storing the cache file.

        This returns two paths for the default cache directory for storing the cache data,
        and the filepath for the dbcollection.json file where the metadata for all datasets
        is stored.

        This paths were designed to work on windows, linx, mac, etc.

        Parameters
        ----------
            None

        Returns
        -------
            None

        Raises
        ------
            None
        """
         # cache directory path (should work for all platforms)
        self.cache_dir = os.path.expanduser("~")

        # cache file path
        self.cache_fname = os.path.join(self.cache_dir, '.dbcollection.json')

        # default paths
        self.default_cache_dir = os.path.join(self.cache_dir, 'dbcollection')
        self.default_data_dir = self.default_cache_dir

    def read_data_cache_file(self):
        """
        Read the cache file data to memory.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Data structure of the cache (file).

        Raises
        ------
        IOError
            If the file cannot be opened.
        """
        try:
            with open(self.cache_fname, 'r') as json_data:
                return json.load(json_data)
        except IOError:
            raise IOError('Unable to open file: ' + self.cache_fname)

    def read_data_cache(self):
        """
        Load data from the dbcollection cache file.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Data structure of the cache (file).

        Raises
        ------
            None
        """
        # check if file exists
        if os.path.exists(self.cache_fname):
            return self.read_data_cache_file()
        else:
            return self.empty_data()

    def write_data_cache(self, data, fname=None):
        """
        Write data to the dbcollection cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        fname : str
            File path+name to store the cache data.

        Returns
        -------
            None

        Raises
        ------
        IOError
            If the file cannot be opened.
        """
        filename = fname or self.cache_fname
        try:
            with open(filename, 'w') as file_cache:
                json.dump(data, file_cache, ensure_ascii=False)
        except IOError:
            raise IOError('Unable to open file: ' + filename)

    def empty_data(self):
        """
        Returns an empty template of the cache data structure.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Dummy data structure.

        Raises
        ------
            None
        """
        return {
            "info": {
                "default_cache_dir": self.default_cache_dir,
		        "default_data_dir": self.default_data_dir
            },
            "dataset": {}
        }

    def create_cache_file_disk(self, fname=None):
        """
        Initialize the dbcollection cache file with empty data.

        Parameters
        ----------
        fname : str
            File name+path on disk.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        self.write_data_cache(self.empty_data(), fname)

    def os_remove(self, fname):
        """
        Remove a file/directory from disk.

        Parameters
        ----------
        fname : str
            File name+path on disk.

        Returns
        -------
            None

        Raises
        ------
        OSError
            If the file does not exist.
        """
        try:
            os.remove(fname)
        except OSError as err:
            if err.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
                raise OSError('Unable to remove: ' + fname)

    def delete_entry(self, name):
        """
        Delete a dataset entry from a category dictionary.

        Parameters
        ----------
        name : str
            Dataset name.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        self.data['dataset'][self.get_category(name)].pop(name)

    def delete_dataset_cache(self, name):
        """
        Delete the cache data from disk of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # get cache dir path
        cache_dir_path = self.get_data_from_field(name, 'cache_dir')

        # remove cache dir
        self.os_remove(cache_dir_path)

        # remove entry from the data
        self.delete_entry(name)

        # write updated data to file
        self.write_data_cache(self.data)

    def check_dataset_name(self, name):
        """
        Check if the dataset name exists in the available dictionary keys.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        bool
            If the dataset exists, returns True.
            Else return False.

        Raises
        ------
            None
        """
        try:
            return name in self.data['dataset'][self.get_category(name)].keys()
        except KeyError:
            return False

    def get_data_from_field(self, name, field):
        """
        Get data from a field of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        field : str
            Field name identifier.

        Returns
        -------
        dict
            Cache's data structure.

        Raises
        ------
            None
        """
        return self.data['dataset'][self.get_category(name)][name][field]

    def change_field(self, name, field, val):
        """
        Change the data of a field of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        field : str
            Field name identifier.
        val : <any>
            New value.

        Returns
        -------
        dict
            Cache's data structure.

        Raises
        ------
            None
        """
        self.data['dataset'][self.get_category(name)][name][field] = val

    def add_data(self, name, new_info):
        """
        Adds/appends a new category/dataset to the cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        new_info : dict
            New data.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # get stored data
        old_info = self.get_dataset_data(name)

        # append the new data
        old_info['task'].update(new_info['task'])

    def delete_dataset(self, name, delete_data=False):
        """
        Delete a dataset from disk/cache.

        Parameters
        ----------
        name : str
            Name of the dataset.
        delete_data : bool
            Flag indicating if the data's directory has to be deleted (or skip it).

        Returns
        -------
            None

        Raises
        ------
            None
        """
        dset_paths = self.get_dataset_storage_paths(name)

        # remove cache directory
        self.delete_dataset_cache(name)

        # delete data from disk
        if delete_data:
            self.os_remove(dset_paths['data_dir'])

    def get_category(self, name):
        """
        Returns the dataset category name of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        str
            Returns the category name if found, else returns None.

        Raises
        ------
            None
        """
        for category in self.data['dataset'].keys():
            if name in self.data['dataset'][category].keys():
                return category
        return None

    def exists_dataset(self, name):
        """
        Check if a dataset exists for loading.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        bool
            Return true if the category exists, else return False.

        Raises
        ------
            None
        """
        # check if the dataset exists in the cache file
        try:
            return name in self.data['dataset'][self.get_category(name)]
        except KeyError:
            return False

    def exists_task(self, name, task):
        """
        Check if a task of a dataset exists in the cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        task : str
            Name of the task.

        Returns
        -------
        bool
            Return true if the task exists, else return False.

        Raises
        ------
            None
        """
        try:
            return task in self.data['dataset'][self.get_category(name)][name]['task'].keys()
        except KeyError:
            return False

    def get_dataset_storage_paths(self, name):
        """
        Get dataset save/load path.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        dict
            Dataset's data+cache paths.

        Raises
        ------
            None
        """
        if self.exists_dataset(name):
            return {
                "cache_dir": self.get_data_from_field(name, "cache_dir"),
                "data_dir": self.get_data_from_field(name, "data_dir")
            }
        else:
            return {
                "cache_dir": os.path.join(self.default_cache_dir, name, 'cache'),
                "data_dir": os.path.join(self.default_data_dir, name, 'data')
            }

    def get_dataset_data(self, name):
        """
        Fetches the cache data of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        dict
            Cache data structure of a particular dataset.

        Raises
        ------
            None
        """
        return self.data['dataset'][self.get_category(name)][name]

    def get_cache_path(self, name, task):
        """
        Return the cache path of a specific task.

        Parameters
        ----------
        name : str
            Name of the dataset.
        task : str
            Name of the task.

        Returns
        -------
        str
            Cache name+path of a specific task.

        Raises
        ------
            None
        """
        # fetch cache data of the dataset
        cache_data = self.get_dataset_data(name)

        # fetch the path of the metadata file for this task
        return cache_data['task'][task]

    def update(self, name, data_dir, cache_dir, cache_info):
        """
        Update the cache file with new/updated data for a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        data_dir : str
            Dataset directory on disk where all data is stored.
        cache_dir : str
            Cache's directory on disk where the metadata files are store.
        cache_info : dict
            New task + path information.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # build info dictionary
        new_info_dict = {
            "data_dir": data_dir,
            "cache_dir": cache_dir,
            "task": cache_info
        }

        # update data with the new info
        self.add_data(name, new_info_dict)

        # write to file
        self.write_data_cache(self.data)
<commit_msg>fixed docstring<commit_after>"""
Class to manage the dbcollection.json cache file.
"""

import os
import errno
import json

class CacheManager:
    """ Class to manage the dbcollection cache data """

    def __init__(self):
        """
        Initialize class.

        Parameters
        ----------
            None
        """

        # setup cache paths
        self.setup_paths()

        # create cache file (if it does not exist)
        if not os.path.exists(self.cache_fname):
            self.create_cache_file_disk(self.cache_fname)

        # load cache data file
        self.data = self.read_data_cache()

    def setup_paths(self):
        """Setup the cache/data directories for storing the cache file.

        This returns two paths for the default cache directory for storing the cache data,
        and the filepath for the dbcollection.json file where the metadata for all datasets
        is stored.

        This paths were designed to work on windows, linx, mac, etc.

        Parameters
        ----------
            None

        Returns
        -------
            None

        Raises
        ------
            None
        """
         # cache directory path (should work for all platforms)
        self.cache_dir = os.path.expanduser("~")

        # cache file path
        self.cache_fname = os.path.join(self.cache_dir, '.dbcollection.json')

        # default paths
        self.default_cache_dir = os.path.join(self.cache_dir, 'dbcollection')
        self.default_data_dir = self.default_cache_dir

    def read_data_cache_file(self):
        """
        Read the cache file data to memory.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Data structure of the cache (file).

        Raises
        ------
        IOError
            If the file cannot be opened.
        """
        try:
            with open(self.cache_fname, 'r') as json_data:
                return json.load(json_data)
        except IOError:
            raise IOError('Unable to open file: ' + self.cache_fname)

    def read_data_cache(self):
        """
        Load data from the dbcollection cache file.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Data structure of the cache (file).

        Raises
        ------
            None
        """
        # check if file exists
        if os.path.exists(self.cache_fname):
            return self.read_data_cache_file()
        else:
            return self.empty_data()

    def write_data_cache(self, data, fname=None):
        """
        Write data to the dbcollection cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        fname : str
            File path+name to store the cache data.

        Returns
        -------
            None

        Raises
        ------
        IOError
            If the file cannot be opened.
        """
        filename = fname or self.cache_fname
        try:
            with open(filename, 'w') as file_cache:
                json.dump(data, file_cache, ensure_ascii=False)
        except IOError:
            raise IOError('Unable to open file: ' + filename)

    def empty_data(self):
        """
        Returns an empty template of the cache data structure.

        Parameters
        ----------
            None

        Returns
        -------
        dict
            Dummy data structure.

        Raises
        ------
            None
        """
        return {
            "info": {
                "default_cache_dir": self.default_cache_dir,
		        "default_data_dir": self.default_data_dir
            },
            "dataset": {}
        }

    def create_cache_file_disk(self, fname=None):
        """
        Initialize the dbcollection cache file with empty data.

        Parameters
        ----------
        fname : str
            File name+path on disk.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        self.write_data_cache(self.empty_data(), fname)

    def os_remove(self, fname):
        """
        Remove a file/directory from disk.

        Parameters
        ----------
        fname : str
            File name+path on disk.

        Returns
        -------
            None

        Raises
        ------
        OSError
            If the file does not exist.
        """
        try:
            os.remove(fname)
        except OSError as err:
            if err.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
                raise OSError('Unable to remove: ' + fname)

    def delete_entry(self, name):
        """
        Delete a dataset entry from a category dictionary.

        Parameters
        ----------
        name : str
            Dataset name.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        self.data['dataset'][self.get_category(name)].pop(name)

    def delete_dataset_cache(self, name):
        """
        Delete the cache data from disk of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # get cache dir path
        cache_dir_path = self.get_data_from_field(name, 'cache_dir')

        # remove cache dir
        self.os_remove(cache_dir_path)

        # remove entry from the data
        self.delete_entry(name)

        # write updated data to file
        self.write_data_cache(self.data)

    def check_dataset_name(self, name):
        """
        Check if the dataset name exists in the available dictionary keys.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        bool
            If the dataset exists, returns True.
            Else return False.

        Raises
        ------
            None
        """
        try:
            return name in self.data['dataset'][self.get_category(name)].keys()
        except KeyError:
            return False

    def get_data_from_field(self, name, field):
        """
        Get data from a field of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        field : str
            Field name identifier.

        Returns
        -------
        dict
            Cache's data structure.

        Raises
        ------
            None
        """
        return self.data['dataset'][self.get_category(name)][name][field]

    def change_field(self, name, field, val):
        """
        Change the data of a field of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        field : str
            Field name identifier.
        val : <any>
            New value.

        Returns
        -------
        dict
            Cache's data structure.

        Raises
        ------
            None
        """
        self.data['dataset'][self.get_category(name)][name][field] = val

    def add_data(self, name, new_info):
        """
        Adds/appends a new category/dataset to the cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        new_info : dict
            New data.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # get stored data
        old_info = self.get_dataset_data(name)

        # append the new data
        old_info['task'].update(new_info['task'])

    def delete_dataset(self, name, delete_data=False):
        """
        Delete a dataset from disk/cache.

        Parameters
        ----------
        name : str
            Name of the dataset.
        delete_data : bool
            Flag indicating if the data's directory has to be deleted (or skip it).

        Returns
        -------
            None

        Raises
        ------
            None
        """
        dset_paths = self.get_dataset_storage_paths(name)

        # remove cache directory
        self.delete_dataset_cache(name)

        # delete data from disk
        if delete_data:
            self.os_remove(dset_paths['data_dir'])

    def get_category(self, name):
        """
        Returns the dataset category name of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        str
            Returns the category name if found, else returns None.

        Raises
        ------
            None
        """
        for category in self.data['dataset'].keys():
            if name in self.data['dataset'][category].keys():
                return category
        return None

    def exists_dataset(self, name):
        """
        Check if a dataset exists for loading.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        bool
            Return true if the category exists, else return False.

        Raises
        ------
            None
        """
        # check if the dataset exists in the cache file
        try:
            return name in self.data['dataset'][self.get_category(name)]
        except KeyError:
            return False

    def exists_task(self, name, task):
        """
        Check if a task of a dataset exists in the cache file.

        Parameters
        ----------
        name : str
            Name of the dataset.
        task : str
            Name of the task.

        Returns
        -------
        bool
            Return true if the task exists, else return False.

        Raises
        ------
            None
        """
        try:
            return task in self.data['dataset'][self.get_category(name)][name]['task'].keys()
        except KeyError:
            return False

    def get_dataset_storage_paths(self, name):
        """
        Get dataset save/load path.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        dict
            Dataset's data+cache paths.

        Raises
        ------
            None
        """
        if self.exists_dataset(name):
            return {
                "cache_dir": self.get_data_from_field(name, "cache_dir"),
                "data_dir": self.get_data_from_field(name, "data_dir")
            }
        else:
            return {
                "cache_dir": os.path.join(self.default_cache_dir, name, 'cache'),
                "data_dir": os.path.join(self.default_data_dir, name, 'data')
            }

    def get_dataset_data(self, name):
        """
        Fetches the cache data of a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.

        Returns
        -------
        dict
            Cache data structure of a particular dataset.

        Raises
        ------
            None
        """
        return self.data['dataset'][self.get_category(name)][name]

    def get_cache_path(self, name, task):
        """
        Return the cache path of a specific task.

        Parameters
        ----------
        name : str
            Name of the dataset.
        task : str
            Name of the task.

        Returns
        -------
        str
            Cache name+path of a specific task.

        Raises
        ------
            None
        """
        # fetch cache data of the dataset
        cache_data = self.get_dataset_data(name)

        # fetch the path of the metadata file for this task
        return cache_data['task'][task]

    def update(self, name, data_dir, cache_dir, cache_info):
        """
        Update the cache file with new/updated data for a dataset.

        Parameters
        ----------
        name : str
            Name of the dataset.
        data_dir : str
            Dataset directory on disk where all data is stored.
        cache_dir : str
            Cache's directory on disk where the metadata files are store.
        cache_info : dict
            New task + path information.

        Returns
        -------
            None

        Raises
        ------
            None
        """
        # build info dictionary
        new_info_dict = {
            "data_dir": data_dir,
            "cache_dir": cache_dir,
            "task": cache_info
        }

        # update data with the new info
        self.add_data(name, new_info_dict)

        # write to file
        self.write_data_cache(self.data)
<|endoftext|>


<commit_before># Copyright: 2014 Dennis Schmalacker <github@progde.de>
# License: BSD 2-clause, see LICENSE for details.

import errno
import base64
import time
from io import BytesIO

from . import rest_api
from flask.views import MethodView
from flask import Response, current_app, request, make_response, url_for, jsonify, stream_with_context

from ..utils.name import ItemName
from ..utils.http import ContentRange, DownloadRange
from ..utils.upload import Upload, background_compute_hash
from ..utils.permissions import *

class ItemUploadView(MethodView):
    def post(self):
        """
        Upload file via REST-API. Chunked Upload is supported.

        HTTP Headers that need to be given:
        * Content-Type: The type of the file that is being uploaded.
            If this is not given filetype will be 'application/octet-stream'
        * Content-Length: The total size of the file to be uploaded.
        * Content-Filename: The filename of the file. This will be used when downloading.
        * Content-Range: The Content-Range of the Chunk that is currently being uploaded.
            Follows the HTTP-Header Specifications.
        * Transaction-Id: The Transaction-Id for Chunked Uploads.
            Needs to be delivered when uploading in chunks (after the first chunk).

        To start an upload, the HTTP Headers need to be delivered.
        The body of the request needs to be the base64 encoded file contents.
        Content-Length is the original file size before base64 encoding.
        Content-Range follows the same logic.
        After the first chunk is uploaded, bepasty will return the Transaction-Id to continue the upload.
        Deliver the Transaction-Id and the correct Content-Range to continue upload.
        After the file is completely uploaded, the file will be marked as complete and
        a 201 HTTP Status will be returned.
        The Content-Location Header will contain the api url to the uploaded Item.

        If the file size exceeds the permitted size, the upload will be aborted. This will be checked twice.
        The first check is the provided Content-Length. The second is the actual file size on the server.
        """
        if not may(CREATE):
            return 'Missing Permissions', 403

        # Collect all expected data from the Request
        file_type = request.headers.get("Content-Type")
        file_size = request.headers.get("Content-Length")
        file_name = request.headers.get("Content-Filename")

        # Check the file size from Request
        Upload.filter_size(file_size)

        # Check if Transaction-ID is available for continued upload
        if not request.headers.get("Transaction-Id"):
            # Create ItemName and empty file in Storage
            name = ItemName.create()
            item = current_app.storage.create(name, 0)

            # set max lifetime
            maxlife_unit = request.form.get('maxlife-unit')
            maxlife_value = int(request.form.get('maxlife-value'))
            units = {
                'forever': -1,
                'minutes': maxlife_value * 60,
                'hours': maxlife_value * 60 * 60,
                'days': maxlife_value * 60 * 60 * 24,
                'weeks': maxlife_value * 60 * 60 * 24 * 7,
                'years': maxlife_value * 60 * 60 * 24 * 365
            }
            maxlife_timestamp = time.time() + units[maxlife_unit]\
                if units[maxlife_unit] > 0 else units[maxlife_unit]
            # Fill meta with data from Request
            Upload.meta_new(item, 0, file_name, file_type,
                            'application/octet-stream',
                            name, maxlife_stamp=maxlife_timestamp)
        else:
            # Get file name from Transaction-ID and open from Storage
            name = base64.b64decode(request.headers.get("Transaction-Id"))
            item = current_app.storage.openwrite(name)

        # Check the actual size of the file on the server against limit
        # Either 0 if new file or n bytes of already uploaded file
        Upload.filter_size(item.data.size)

        # Check Content-Range. Needs to be specified, even if only one chunk
        if not request.headers.get("Content-Range"):
            return 'Content-Range not specified', 400

        # Get Content-Range and check if Range is consistent with server state
        file_range = ContentRange.from_request()
        if not item.data.size == file_range.begin:
            return ('Content-Range inconsistent. Last byte on Server: %d' % item.data.size), 409

        # Decode Base64 encoded request data
        raw_data = base64.b64decode(request.data)
        file_data = BytesIO(raw_data)

        # Write data chunk to item
        Upload.data(item, file_data, len(raw_data), file_range.begin)

        # Make a Response and create Transaction-ID from ItemName
        response = make_response()
        response.headers["Transaction-Id"] = base64.b64encode(name)
        response.status = '200'

        # Check if file is completely uploaded and set meta
        if file_range.is_complete:
            Upload.meta_complete(item, '')
            item.meta['size'] = item.data.size
            background_compute_hash(current_app.storage, name)
            # Set status 'successful' and return the new URL for the uploaded file
            response.status = '201'
            response.headers["Content-Location"] = url_for('bepasty_rest.items_detail', name=name)

        item.close()
        return response

class ItemDetailView(MethodView):
    def get(self, name):
        if not may(READ):
            return 'Missing Permissions', 403

        with current_app.storage.open(name) as item:
            return jsonify({'uri': url_for('bepasty_rest.items_detail', name=name),
                            'file-meta': dict(item.meta)})

class ItemDownloadView(MethodView):
    content_disposition = 'attachment'

    def get(self, name):
        if not may(READ):
            return 'Missing Permissions', 403

        try:
            item = current_app.storage.open(name)
        except (OSError, IOError) as e:
            if e.errno == errno.ENOENT:
                return 'File not found', 404
            raise

        if not item.meta.get('unlocked'):
            error = 'File Locked.'
        elif not item.meta.get('complete'):
            error = 'Upload incomplete. Try again later.'
        else:
            error = None
        if error:
            item.close()
            return error, 403

        request_range = DownloadRange.from_request()
        if not request_range:
            range_end = item.data.size
            range_begin = 0
        else:
            if request_range.end == -1:
                range_end = item.data.size
            else:
                range_end = min(request_range.end, item.data.size)
            range_begin = request_range.begin

        def stream(begin, end):
            offset = max(0, begin)
            with item as _item:
                while offset < end:
                    buf = _item.data.read(16 * 1024, offset)
                    offset += len(buf)
                    yield buf

        ret = Response(stream_with_context(stream(range_begin, range_end)))
        ret.headers['Content-Disposition'] = '{}; filename="{}"'.format(
            self.content_disposition, item.meta['filename'])
        ret.headers['Content-Length'] = (range_end - range_begin) + 1
        ret.headers['Content-Type'] = item.meta['type']  # 'application/octet-stream'
        ret.status = '200'
        ret.headers['Content-Range'] = ('bytes %d-%d/%d' % (range_begin, range_end, item.data.size))
        return ret

rest_api.add_url_rule('/items', view_func=ItemUploadView.as_view('items'))
rest_api.add_url_rule('/items/<itemname:name>', view_func=ItemDetailView.as_view('items_detail'))
rest_api.add_url_rule('/items/<itemname:name>/download', view_func=ItemDownloadView.as_view('items_download'))
<commit_msg>Removed more unused code for the expiration date feature<commit_after># Copyright: 2014 Dennis Schmalacker <github@progde.de>
# License: BSD 2-clause, see LICENSE for details.

import errno
import base64
import time
from io import BytesIO

from . import rest_api
from flask.views import MethodView
from flask import Response, current_app, request, make_response, url_for, jsonify, stream_with_context

from ..utils.name import ItemName
from ..utils.http import ContentRange, DownloadRange
from ..utils.upload import Upload, background_compute_hash
from ..utils.permissions import *
from ..utils.date_funcs import time_unit_to_sec

class ItemUploadView(MethodView):
    def post(self):
        """
        Upload file via REST-API. Chunked Upload is supported.

        HTTP Headers that need to be given:
        * Content-Type: The type of the file that is being uploaded.
            If this is not given filetype will be 'application/octet-stream'
        * Content-Length: The total size of the file to be uploaded.
        * Content-Filename: The filename of the file. This will be used when downloading.
        * Content-Range: The Content-Range of the Chunk that is currently being uploaded.
            Follows the HTTP-Header Specifications.
        * Transaction-Id: The Transaction-Id for Chunked Uploads.
            Needs to be delivered when uploading in chunks (after the first chunk).

        To start an upload, the HTTP Headers need to be delivered.
        The body of the request needs to be the base64 encoded file contents.
        Content-Length is the original file size before base64 encoding.
        Content-Range follows the same logic.
        After the first chunk is uploaded, bepasty will return the Transaction-Id to continue the upload.
        Deliver the Transaction-Id and the correct Content-Range to continue upload.
        After the file is completely uploaded, the file will be marked as complete and
        a 201 HTTP Status will be returned.
        The Content-Location Header will contain the api url to the uploaded Item.

        If the file size exceeds the permitted size, the upload will be aborted. This will be checked twice.
        The first check is the provided Content-Length. The second is the actual file size on the server.
        """
        if not may(CREATE):
            return 'Missing Permissions', 403

        # Collect all expected data from the Request
        file_type = request.headers.get("Content-Type")
        file_size = request.headers.get("Content-Length")
        file_name = request.headers.get("Content-Filename")

        # Check the file size from Request
        Upload.filter_size(file_size)

        # Check if Transaction-ID is available for continued upload
        if not request.headers.get("Transaction-Id"):
            # Create ItemName and empty file in Storage
            name = ItemName.create()
            item = current_app.storage.create(name, 0)

            # set max lifetime
            maxlife_unit = request.form.get('maxlife-unit')
            maxlife_value = int(request.form.get('maxlife-value'))
            maxlife_timestamp = time.time() + time_unit_to_sec(maxlife_value, maxlife_unit)
            # Fill meta with data from Request
            Upload.meta_new(item, 0, file_name, file_type,
                            'application/octet-stream',
                            name, maxlife_stamp=maxlife_timestamp)
        else:
            # Get file name from Transaction-ID and open from Storage
            name = base64.b64decode(request.headers.get("Transaction-Id"))
            item = current_app.storage.openwrite(name)

        # Check the actual size of the file on the server against limit
        # Either 0 if new file or n bytes of already uploaded file
        Upload.filter_size(item.data.size)

        # Check Content-Range. Needs to be specified, even if only one chunk
        if not request.headers.get("Content-Range"):
            return 'Content-Range not specified', 400

        # Get Content-Range and check if Range is consistent with server state
        file_range = ContentRange.from_request()
        if not item.data.size == file_range.begin:
            return ('Content-Range inconsistent. Last byte on Server: %d' % item.data.size), 409

        # Decode Base64 encoded request data
        raw_data = base64.b64decode(request.data)
        file_data = BytesIO(raw_data)

        # Write data chunk to item
        Upload.data(item, file_data, len(raw_data), file_range.begin)

        # Make a Response and create Transaction-ID from ItemName
        response = make_response()
        response.headers["Transaction-Id"] = base64.b64encode(name)
        response.status = '200'

        # Check if file is completely uploaded and set meta
        if file_range.is_complete:
            Upload.meta_complete(item, '')
            item.meta['size'] = item.data.size
            background_compute_hash(current_app.storage, name)
            # Set status 'successful' and return the new URL for the uploaded file
            response.status = '201'
            response.headers["Content-Location"] = url_for('bepasty_rest.items_detail', name=name)

        item.close()
        return response

class ItemDetailView(MethodView):
    def get(self, name):
        if not may(READ):
            return 'Missing Permissions', 403

        with current_app.storage.open(name) as item:
            return jsonify({'uri': url_for('bepasty_rest.items_detail', name=name),
                            'file-meta': dict(item.meta)})

class ItemDownloadView(MethodView):
    content_disposition = 'attachment'

    def get(self, name):
        if not may(READ):
            return 'Missing Permissions', 403

        try:
            item = current_app.storage.open(name)
        except (OSError, IOError) as e:
            if e.errno == errno.ENOENT:
                return 'File not found', 404
            raise

        if not item.meta.get('unlocked'):
            error = 'File Locked.'
        elif not item.meta.get('complete'):
            error = 'Upload incomplete. Try again later.'
        else:
            error = None
        if error:
            item.close()
            return error, 403

        request_range = DownloadRange.from_request()
        if not request_range:
            range_end = item.data.size
            range_begin = 0
        else:
            if request_range.end == -1:
                range_end = item.data.size
            else:
                range_end = min(request_range.end, item.data.size)
            range_begin = request_range.begin

        def stream(begin, end):
            offset = max(0, begin)
            with item as _item:
                while offset < end:
                    buf = _item.data.read(16 * 1024, offset)
                    offset += len(buf)
                    yield buf

        ret = Response(stream_with_context(stream(range_begin, range_end)))
        ret.headers['Content-Disposition'] = '{}; filename="{}"'.format(
            self.content_disposition, item.meta['filename'])
        ret.headers['Content-Length'] = (range_end - range_begin) + 1
        ret.headers['Content-Type'] = item.meta['type']  # 'application/octet-stream'
        ret.status = '200'
        ret.headers['Content-Range'] = ('bytes %d-%d/%d' % (range_begin, range_end, item.data.size))
        return ret

rest_api.add_url_rule('/items', view_func=ItemUploadView.as_view('items'))
rest_api.add_url_rule('/items/<itemname:name>', view_func=ItemDetailView.as_view('items_detail'))
rest_api.add_url_rule('/items/<itemname:name>/download', view_func=ItemDownloadView.as_view('items_download'))
<|endoftext|>


<commit_before>import zlib
import cPickle
import pymongo
import time

client = pymongo.MongoClient("gouda.bucknell.edu")
db = client.cheddar
collection = db.times

count = 0

while(True):
    if count < collection.count():                                  #check if there is more data left
        query = collection.find(snapshot=True).sort('_id',1)
        #first = collection.find().sort("_id",1)[0]["_id"]  #get id of first element
        #data = collection.find({"_id":first+count})['data']         #get data from first+count elements
        while(count < query.count()-1):
            data = query[count]['data']
            pickled = cPickle.dumps(data)
            compressed = zlib.compress(pickled,9)
            update = {}
            update['$set'] = {'data':compressed}
            collection.update_one({'_id':query[count]['_id']},update)
            count += 1
    else:
        time.sleep(300)

<commit_msg>compressor works and is adding to a backup library currently<commit_after>import zlib
import cPickle
import pymongo
import time
import bson

client = pymongo.MongoClient("gouda.bucknell.edu")
db = client.cheddar
collection = db.times
testcol = db.compress

count = 0

while(True):
    size = collection.count();
    if count < size:                                  #check if there is more data left
        responses = collection.find({},{'_id':1})
        ids = []
        for i in responses:
        		ids.append(i['_id'])
        ids.sort()
        ids = ids[0:len(ids)-1]
        #first = collection.find().sort("_id",1)[0]["_id"]  				#get id of first element
        #data = collection.find({"_id":first+count})['data']        #get data from first+count elements
        while(count < len(ids)):
            data = collection.find_one({'_id':ids[count]})['data']
            pickled = cPickle.dumps(data)
            compressed = zlib.compress(pickled,9)
            update = {}
            update['$set'] = {'data':bson.binary.Binary(compressed)}
            testcol.update({'_id':ids[count]},update,upsert=True)
            print ("updating: " + str(ids[count]))
            count += 1
    else:
        time.sleep(300)

<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import matplotlib.gridspec as mgridspec
import matplotlib.ticker as mticker
import matplotlib
import mpl_toolkits.mplot3d as mplot3d
import numpy
import math
import itertools
import multiprocessing
import pysb.integrate

__all__ = ['scatter', 'surf', 'sample']

def scatter(mcmc, mask=True):
    """
    Display a grid of scatter plots for each 2-D projection of an MCMC walk.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        The MCMC object to display.
    mask : bool/int, optional
        If True (default) the annealing phase of the walk will be discarded
        before plotting. If False, nothing will be discarded and all points will
        be plotted. If an integer, specifies the number of steps to be discarded
        from the beginning of the walk.

    """

    # number of dimensions in position vector
    ndims = mcmc.num_estimate
    # vector of booleans indicating accepted MCMC moves
    accepts = mcmc.accepts.copy()
    # mask off the annealing (burn-in) phase, or up to a user-specified step
    if mask is True:
        mask = mcmc.options.anneal_length
    if mask is False:
        mask = 0
    accepts[0:mask] = 0
    # grab position vectors and posterior values from accepted moves
    positions = mcmc.positions[accepts]
    posteriors = mcmc.posteriors[accepts]
    # calculate actual range of values on each dimension
    maxes = positions.max(0)
    mins = positions.min(0)
    ranges = abs(maxes - mins)
    # use 2% of the maximum range as a margin for all scatter plots
    margin = max(ranges) * 0.02
    # calculate upper and lower plot limits based on min/max plus the margin
    lims_top = maxes + margin
    lims_bottom = mins - margin
    # calculate new ranges based on limits
    lim_ranges = abs(lims_top - lims_bottom)

    plt.figure()
    # build a GridSpec which allocates space based on these ranges
    gs = mgridspec.GridSpec(ndims, ndims, width_ratios=lim_ranges,
                            height_ratios=lim_ranges[-1::-1])
    # build an axis locator for each dimension
    locators = []
    for i, r in enumerate(lim_ranges):
        # place ticks on the integers, unless there is no integer within the
        # given dimension's calculated range
        nbins = math.ceil(r) + 1
        want_integer = math.ceil(lims_bottom[i]) <= math.floor(lims_top[i])
        locators.append(mticker.MaxNLocator(nbins=nbins, integer=want_integer))

    fignum = 0
    # reverse the param list along the y axis so we end up with the "origin"
    # (i.e. the first param) at the bottom left instead of the top left. note
    # that y==0 will be along the bottom now, but the figure numbers in the
    # gridspec still begin counting at the top.
    for y, py in reversed(list(enumerate(mcmc.options.estimate_params))):
        for x, px in enumerate(mcmc.options.estimate_params):
            ax = plt.subplot(gs[fignum])
            ax.tick_params(left=False, right=True, top=True, bottom=False,
                           labelleft=False, labelright=False, labeltop=False,
                           labelbottom=False, direction='in')
            ax.yaxis.set_major_locator(locators[y])
            ax.xaxis.set_major_locator(locators[x])
            if x == y:
                # 1-D histograms along the diagonal
                ax.hist(positions[:,x], color='salmon', ec='tomato')
                ax.set_xlim(lims_bottom[x], lims_top[x])
                ax.yaxis.set_major_locator(mticker.NullLocator())
            else:
                # 2-D scatter plots off the diagonal
                ax.plot(positions[:, x], positions[:, y], color='darkblue',
                        alpha=0.2)
                ax.scatter(positions[:, x], positions[:, y], s=1, color='darkblue',
                           alpha=0.2)
                ax.set_xlim(lims_bottom[x], lims_top[x])
                ax.set_ylim(lims_bottom[y], lims_top[y])
            # parameter name labels along left and bottom edge of the grid
            if x == 0:
                ax.set_ylabel(py.name, weight='black', size='large',
                              labelpad=10, rotation='horizontal',
                              horizontalalignment='right')
            if y == 0:
                ax.set_xlabel(px.name, weight='black', size='large',
                              labelpad=10,)
            # tick labels along the right and top edge of the grid
            if x == ndims - 1:
                ax.tick_params('y', labelright=True)
            if y == ndims - 1:
                ax.tick_params('x', labeltop=True)
            # move to next figure in the gridspec
            fignum += 1
    # TODO: would axis('scaled') force the aspect ratio we want?
    plt.show()

def surf(mcmc, dim0, dim1, mask=True, square_aspect=True, margin=0.1,
         gridsize=20):
    """
    Display the posterior of an MCMC walk as a 3-D surface.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        MCMC object to display.
    dim0, dim1 : indices of parameters to display
    mask : bool/int, optional
        If True (default) the annealing phase of the walk will be discarded
        before plotting. If False, nothing will be discarded and all points will
        be plotted. If an integer, specifies the number of steps to be discarded
        from the beginning of the walk.
    square_aspect : bool, optional
        If True (default) the X and Y scales of the plot will be equal, allowing
        for direct comparison of moves in the corresponding parameter axes. If
        False the scales will auto-adjust to fit the data tightly, allowing for
        visualization of the full variance along both axes.
    margin : float, optional
        Fraction of the X and Y ranges to add as padding to the plot. Defaults
        to 0.1.
    """
    # mask off the annealing (burn-in) phase, or up to a user-specified step
    if mask is True:
        mask = mcmc.options.anneal_length
    elif mask is False:
        mask = 0
    # create masked versions of a few vectors of interest
    accepts = mcmc.accepts[mask:]
    rejects = mcmc.rejects[mask:]
    posteriors = mcmc.posteriors[mask:]
    # filter out the position elements we aren't plotting
    positions = mcmc.positions[mask:, (dim0, dim1)]
    # build grid of points for sampling the posterior surface
    pos_min = positions.min(0)
    pos_max = positions.max(0)
    if square_aspect:
        # enforce square aspect ratio in X-Y plane by recomputing pos_min/max
        pos_max_range = numpy.max(pos_max - pos_min)
        pos_mean = numpy.mean([pos_min, pos_max], 0)
        pos_min = pos_mean - pos_max_range / 2
        pos_max = pos_mean + pos_max_range / 2
    margin_offset = (pos_max - pos_min) * margin
    pos_min -= margin_offset
    pos_max += margin_offset
    p0_vals = numpy.linspace(pos_min[0], pos_max[0], gridsize)
    p1_vals = numpy.linspace(pos_min[1], pos_max[1], gridsize)
    p0_mesh, p1_mesh = numpy.meshgrid(p0_vals, p1_vals)
    # calculate posterior value at all gridsize*gridsize points
    posterior_mesh = numpy.empty_like(p0_mesh)
    position_base = numpy.median(mcmc.positions, axis=0)
    # use multiprocessing to make use of multiple cores
    idx_iter = itertools.product(range(gridsize), range(gridsize))
    inputs = ((p0_mesh[i0, i1], p1_mesh[i0, i1]) for i0, i1 in idx_iter)
    inputs = itertools.product([mcmc], [position_base], [dim0], [dim1], inputs)
    try:
        pool = multiprocessing.Pool()
    except KeyboardInterrupt:
        pool.terminate()
        raise
    outputs = pool.map(surf_calc_mesh_pos, inputs)
    pool.close()
    for i0 in range(gridsize):
        for i1 in range(gridsize):
            posterior_mesh[i0, i1] = outputs[i0 * gridsize + i1]
    # plot 3-D surface
    fig = plt.figure()
    ax = fig.gca(projection='3d')
    ax.plot_surface(p0_mesh, p1_mesh, posterior_mesh, rstride=1, cstride=1,
                    cmap=matplotlib.cm.jet, linewidth=0, alpha=0.2)
    ax.plot(positions[accepts,0], positions[accepts,1], posteriors[accepts],
            c='k')
    ax.scatter(positions[rejects,0], positions[rejects,1], posteriors[rejects],
               marker='x', c='k', alpha=0.3)
    ax.set_xlabel('ln(%s)' % mcmc.options.estimate_params[dim0].name)
    ax.set_ylabel('ln(%s)' % mcmc.options.estimate_params[dim1].name)
    ax.set_zlabel('-ln(posterior)')
    plt.show()

def surf_calc_mesh_pos(args):
    mcmc, position_base, dim0, dim1, param_vals = args
    p0_val, p1_val = param_vals
    position = position_base.copy()
    position[dim0] = p0_val
    position[dim1] = p1_val
    return mcmc.calculate_posterior(position)[0]

def sample(mcmc, n, colors, norm_factor=None):
    """
    Display model trajectories of parameter sets sampled from the MCMC walk.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        MCMC object to sample from.
    n : int
        Number of samples.
    norm_factor : vector-like, optional
        Vector of scaling factors, the same length as the number of species in
        the model. Each simulated trajectory will be divided by the respective
        value before plotting. If omitted, trajectories will be rescaled to the
        maximum value of each trajectory across all samples and all time points.

    Notes
    -----
    This is not currently useful for models with many species. It should
    probably plot observables instead (and allow selection of specific ones).

    """
    # FIXME plot observables instead, and allow a list of observables or
    # observable names as an argument.
    plt.figure()
    tspan = mcmc.options.tspan
    accept_positions = mcmc.positions[mcmc.accepts]
    if isinstance(n, int):
        idx = range(n)
    ysamples = numpy.empty((len(idx), len(tspan),
                            len(mcmc.options.model.species)))
    for i in idx:
        ysamples[i] = mcmc.simulate(accept_positions[-(i+1)])
    if norm_factor is None:
        norm_factor = ysamples.max(1).max(0)
    ysamples /= norm_factor
    for yout in ysamples:
        for y, c in zip(yout.T, colors):
            plt.plot(tspan, y, c=c, linewidth=1, alpha=.01)
    true_params = [p.value for p in mcmc.options.estimate_params]
    true_position = numpy.log10(true_params)
    true_yout = mcmc.simulate(true_position) / norm_factor
    plt.plot(tspan, true_yout, c='k', linewidth=2)
    plt.show()
<commit_msg>fix keyboardinterrupt handling in plot.surf<commit_after>import matplotlib.pyplot as plt
import matplotlib.gridspec as mgridspec
import matplotlib.ticker as mticker
import matplotlib
import mpl_toolkits.mplot3d as mplot3d
import numpy
import math
import itertools
import multiprocessing
import pysb.integrate

__all__ = ['scatter', 'surf', 'sample']

def scatter(mcmc, mask=True):
    """
    Display a grid of scatter plots for each 2-D projection of an MCMC walk.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        The MCMC object to display.
    mask : bool/int, optional
        If True (default) the annealing phase of the walk will be discarded
        before plotting. If False, nothing will be discarded and all points will
        be plotted. If an integer, specifies the number of steps to be discarded
        from the beginning of the walk.

    """

    # number of dimensions in position vector
    ndims = mcmc.num_estimate
    # vector of booleans indicating accepted MCMC moves
    accepts = mcmc.accepts.copy()
    # mask off the annealing (burn-in) phase, or up to a user-specified step
    if mask is True:
        mask = mcmc.options.anneal_length
    if mask is False:
        mask = 0
    accepts[0:mask] = 0
    # grab position vectors and posterior values from accepted moves
    positions = mcmc.positions[accepts]
    posteriors = mcmc.posteriors[accepts]
    # calculate actual range of values on each dimension
    maxes = positions.max(0)
    mins = positions.min(0)
    ranges = abs(maxes - mins)
    # use 2% of the maximum range as a margin for all scatter plots
    margin = max(ranges) * 0.02
    # calculate upper and lower plot limits based on min/max plus the margin
    lims_top = maxes + margin
    lims_bottom = mins - margin
    # calculate new ranges based on limits
    lim_ranges = abs(lims_top - lims_bottom)

    plt.figure()
    # build a GridSpec which allocates space based on these ranges
    gs = mgridspec.GridSpec(ndims, ndims, width_ratios=lim_ranges,
                            height_ratios=lim_ranges[-1::-1])
    # build an axis locator for each dimension
    locators = []
    for i, r in enumerate(lim_ranges):
        # place ticks on the integers, unless there is no integer within the
        # given dimension's calculated range
        nbins = math.ceil(r) + 1
        want_integer = math.ceil(lims_bottom[i]) <= math.floor(lims_top[i])
        locators.append(mticker.MaxNLocator(nbins=nbins, integer=want_integer))

    fignum = 0
    # reverse the param list along the y axis so we end up with the "origin"
    # (i.e. the first param) at the bottom left instead of the top left. note
    # that y==0 will be along the bottom now, but the figure numbers in the
    # gridspec still begin counting at the top.
    for y, py in reversed(list(enumerate(mcmc.options.estimate_params))):
        for x, px in enumerate(mcmc.options.estimate_params):
            ax = plt.subplot(gs[fignum])
            ax.tick_params(left=False, right=True, top=True, bottom=False,
                           labelleft=False, labelright=False, labeltop=False,
                           labelbottom=False, direction='in')
            ax.yaxis.set_major_locator(locators[y])
            ax.xaxis.set_major_locator(locators[x])
            if x == y:
                # 1-D histograms along the diagonal
                ax.hist(positions[:,x], color='salmon', ec='tomato')
                ax.set_xlim(lims_bottom[x], lims_top[x])
                ax.yaxis.set_major_locator(mticker.NullLocator())
            else:
                # 2-D scatter plots off the diagonal
                ax.plot(positions[:, x], positions[:, y], color='darkblue',
                        alpha=0.2)
                ax.scatter(positions[:, x], positions[:, y], s=1, color='darkblue',
                           alpha=0.2)
                ax.set_xlim(lims_bottom[x], lims_top[x])
                ax.set_ylim(lims_bottom[y], lims_top[y])
            # parameter name labels along left and bottom edge of the grid
            if x == 0:
                ax.set_ylabel(py.name, weight='black', size='large',
                              labelpad=10, rotation='horizontal',
                              horizontalalignment='right')
            if y == 0:
                ax.set_xlabel(px.name, weight='black', size='large',
                              labelpad=10,)
            # tick labels along the right and top edge of the grid
            if x == ndims - 1:
                ax.tick_params('y', labelright=True)
            if y == ndims - 1:
                ax.tick_params('x', labeltop=True)
            # move to next figure in the gridspec
            fignum += 1
    # TODO: would axis('scaled') force the aspect ratio we want?
    plt.show()

def surf(mcmc, dim0, dim1, mask=True, square_aspect=True, margin=0.1,
         gridsize=20):
    """
    Display the posterior of an MCMC walk as a 3-D surface.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        MCMC object to display.
    dim0, dim1 : indices of parameters to display
    mask : bool/int, optional
        If True (default) the annealing phase of the walk will be discarded
        before plotting. If False, nothing will be discarded and all points will
        be plotted. If an integer, specifies the number of steps to be discarded
        from the beginning of the walk.
    square_aspect : bool, optional
        If True (default) the X and Y scales of the plot will be equal, allowing
        for direct comparison of moves in the corresponding parameter axes. If
        False the scales will auto-adjust to fit the data tightly, allowing for
        visualization of the full variance along both axes.
    margin : float, optional
        Fraction of the X and Y ranges to add as padding to the plot. Defaults
        to 0.1.
    """
    # mask off the annealing (burn-in) phase, or up to a user-specified step
    if mask is True:
        mask = mcmc.options.anneal_length
    elif mask is False:
        mask = 0
    # create masked versions of a few vectors of interest
    accepts = mcmc.accepts[mask:]
    rejects = mcmc.rejects[mask:]
    posteriors = mcmc.posteriors[mask:]
    # filter out the position elements we aren't plotting
    positions = mcmc.positions[mask:, (dim0, dim1)]
    # build grid of points for sampling the posterior surface
    pos_min = positions.min(0)
    pos_max = positions.max(0)
    if square_aspect:
        # enforce square aspect ratio in X-Y plane by recomputing pos_min/max
        pos_max_range = numpy.max(pos_max - pos_min)
        pos_mean = numpy.mean([pos_min, pos_max], 0)
        pos_min = pos_mean - pos_max_range / 2
        pos_max = pos_mean + pos_max_range / 2
    margin_offset = (pos_max - pos_min) * margin
    pos_min -= margin_offset
    pos_max += margin_offset
    p0_vals = numpy.linspace(pos_min[0], pos_max[0], gridsize)
    p1_vals = numpy.linspace(pos_min[1], pos_max[1], gridsize)
    p0_mesh, p1_mesh = numpy.meshgrid(p0_vals, p1_vals)
    # calculate posterior value at all gridsize*gridsize points
    posterior_mesh = numpy.empty_like(p0_mesh)
    position_base = numpy.median(mcmc.positions, axis=0)
    # use multiprocessing to make use of multiple cores
    idx_iter = itertools.product(range(gridsize), range(gridsize))
    inputs = ((p0_mesh[i0, i1], p1_mesh[i0, i1]) for i0, i1 in idx_iter)
    inputs = itertools.product([mcmc], [position_base], [dim0], [dim1], inputs)
    try:
        pool = multiprocessing.Pool()
        outputs = pool.map(surf_calc_mesh_pos, inputs)
        pool.close()
    except KeyboardInterrupt:
        pool.terminate()
        raise
    for i0 in range(gridsize):
        for i1 in range(gridsize):
            posterior_mesh[i0, i1] = outputs[i0 * gridsize + i1]
    # plot 3-D surface
    fig = plt.figure()
    ax = fig.gca(projection='3d')
    ax.plot_surface(p0_mesh, p1_mesh, posterior_mesh, rstride=1, cstride=1,
                    cmap=matplotlib.cm.jet, linewidth=0, alpha=0.2)
    ax.plot(positions[accepts,0], positions[accepts,1], posteriors[accepts],
            c='k')
    ax.scatter(positions[rejects,0], positions[rejects,1], posteriors[rejects],
               marker='x', c='k', alpha=0.3)
    ax.set_xlabel('ln(%s)' % mcmc.options.estimate_params[dim0].name)
    ax.set_ylabel('ln(%s)' % mcmc.options.estimate_params[dim1].name)
    ax.set_zlabel('-ln(posterior)')
    plt.show()

def surf_calc_mesh_pos(args):
    try:
        mcmc, position_base, dim0, dim1, param_vals = args
        p0_val, p1_val = param_vals
        position = position_base.copy()
        position[dim0] = p0_val
        position[dim1] = p1_val
        return mcmc.calculate_posterior(position)[0]
    except KeyboardInterrupt:
        raise RuntimeError()

def sample(mcmc, n, colors, norm_factor=None):
    """
    Display model trajectories of parameter sets sampled from the MCMC walk.

    Parameters
    ----------
    mcmc : bayessb.MCMC
        MCMC object to sample from.
    n : int
        Number of samples.
    norm_factor : vector-like, optional
        Vector of scaling factors, the same length as the number of species in
        the model. Each simulated trajectory will be divided by the respective
        value before plotting. If omitted, trajectories will be rescaled to the
        maximum value of each trajectory across all samples and all time points.

    Notes
    -----
    This is not currently useful for models with many species. It should
    probably plot observables instead (and allow selection of specific ones).

    """
    # FIXME plot observables instead, and allow a list of observables or
    # observable names as an argument.
    plt.figure()
    tspan = mcmc.options.tspan
    accept_positions = mcmc.positions[mcmc.accepts]
    if isinstance(n, int):
        idx = range(n)
    ysamples = numpy.empty((len(idx), len(tspan),
                            len(mcmc.options.model.species)))
    for i in idx:
        ysamples[i] = mcmc.simulate(accept_positions[-(i+1)])
    if norm_factor is None:
        norm_factor = ysamples.max(1).max(0)
    ysamples /= norm_factor
    for yout in ysamples:
        for y, c in zip(yout.T, colors):
            plt.plot(tspan, y, c=c, linewidth=1, alpha=.01)
    true_params = [p.value for p in mcmc.options.estimate_params]
    true_position = numpy.log10(true_params)
    true_yout = mcmc.simulate(true_position) / norm_factor
    plt.plot(tspan, true_yout, c='k', linewidth=2)
    plt.show()
<|endoftext|>


<commit_before>import os
import yaml
import cerberus
import pprint

def open_yaml_file(file_path):
    """
    Open yaml file and return dictionary of values
    :param file_path: file path to yaml file
    :return: dictionary of values from yaml file
    """
    __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    with open(os.path.join(__location__, file_path), 'r') as input_yaml:
        return yaml.safe_load(input_yaml)

def main():
    schema = open_yaml_file('../../amazonia/schemas/cerberus_schema.yaml')
    data = open_yaml_file('../../amazonia/defaults.yaml')
    pp = pprint.PrettyPrinter(indent=4)
    pp.pprint(schema)

    validator = cerberus.Validator()

    validator.validate(data, schema)

if __name__ == '__main__':
    main()
<commit_msg>Removing cerberus testbed<commit_after><|endoftext|>