<commit_before># Standard Libary
import logging

from email_validator import (
    EmailNotValidError,
    validate_email,
)

# Django
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.utils import encoding

# First-Party
from api.models import (
    Group,
    Member,
    Organization,
    Person,
    User,
)
# Remote
from bhs.models import (
    Role,
    SMJoin,
)

log = logging.getLogger('updater')

def update_or_create_person_from_human(human):
    first_name = human.first_name.strip()
    middle_name = human.middle_name.strip()
    last_name = human.last_name.strip()
    nick_name = human.nick_name.replace("'", "").replace('"', '').replace("(", "").replace(")", "").strip()
    if not first_name:
        first_name = None
    if not middle_name:
        middle_name = None
    if not last_name:
        last_name = None
    if nick_name and (nick_name != first_name):
        nick_name = "({0})".format(nick_name)
    else:
        nick_name = None
    name = " ".join(
        map(
            (lambda x: encoding.smart_text(x)),
            filter(
                None, [
                    first_name,
                    middle_name,
                    last_name,
                    nick_name,
                ]
            )
        )
    )
    try:
        v = validate_email(human.email.strip())
        email = v["email"].lower()
    except EmailNotValidError as e:
        email = None
    birth_date = human.birth_date
    if human.phone:
        phone = human.phone
    else:
        phone = ''
    if human.cell_phone:
        cell_phone = human.cell_phone
    else:
        cell_phone = ''
    if human.work_phone:
        work_phone = human.work_phone
    else:
        work_phone = ''
    is_bhs = True
    bhs_id = human.bhs_id
    if human.sex:
        if human.sex.casefold() == 'male'.casefold():
            gender = 10
        elif human.sex.casefold() == 'female'.casefold():
            gender = 20
        else:
            gender = None
    else:
        gender = None
    if human.primary_voice_part:
        if human.primary_voice_part.casefold() == 'Tenor'.casefold():
            part = 1
        elif human.primary_voice_part.casefold() == 'Lead'.casefold():
            part = 2
        elif human.primary_voice_part.casefold() == 'Baritone'.casefold():
            part = 3
        elif human.primary_voice_part.casefold() == 'Bass'.casefold():
            part = 4
        else:
            part = None
    else:
        part = None
    defaults = {
        'name': name,
        'email': email,
        'birth_date': birth_date,
        'phone': phone,
        'cell_phone': cell_phone,
        'work_phone': work_phone,
        'is_bhs': is_bhs,
        'bhs_id': bhs_id,
        'gender': gender,
        'part': part,
    }
    try:
        person, created = Person.objects.update_or_create(
            bhs_pk=human.id,
            defaults=defaults,
        )
        if created and email:
            User.objects.create_user(
                person=person,
                is_active=False,
            )
        log.info("{0}; {1}".format(person, created))
    except IntegrityError as e:
        log.error("{0} {1}".format(e, human))
        return

def update_or_create_group_from_structure(structure):
    kind_map = {
        'chapter': 32,
        'quartet': 41,
    }
    try:
        kind = kind_map[structure.kind]
    except KeyError as e:
        log.error("{0} {1}".format(e, structure))
        return
    status_map = {
        'active': 10,
        'active-licensed': 10,
        'pending': 0,
        'pending-voluntary': 0,
        'expired': -10,
        'closed-revoked': -10,
        'closed-merged': -10,
        'suspended-membership': -10,
        'cancelled': -10,
        'lapsed': -10,
        'closed-voluntary': -10,
        'expelled': -10,
        'suspended': -10,
    }
    if kind == 41:
        if structure.name:
            name = structure.name.strip()
        else:
            if not structure.preferred_name:
                return
            name = "{0} (NAME APPROVAL PENDING)".format(
                structure.preferred_name.strip()
            )
    else:
        name = "{0} [{1}]".format(
            structure.chorus_name.strip(),
            structure.name.strip(),
        )
    status = status_map[str(structure.status)]
    if kind == 32:
        code = structure.chapter_code
    else:
        code = ''
    start_date = structure.established_date
    try:
        v = validate_email(structure.email)
        email = v["email"].lower()
    except EmailNotValidError:
        email = ''
    if structure.phone:
        phone = structure.phone
    else:
        phone = ''
    is_bhs = True
    bhs_id = structure.bhs_id
    defaults = {
        'name': name,
        'status': status,
        'kind': kind,
        'code': code,
        'start_date': start_date,
        'email': email,
        'phone': phone,
        'is_bhs': is_bhs,
        'bhs_id': bhs_id,
    }
    try:
        group, created = Group.objects.update_or_create(
            bhs_pk=structure.id,
            defaults=defaults,
        )
        log.info("{0}; {1}".format(group, created))
    except IntegrityError as e:
        log.error("{0} {1}".format(e, group))
        return
    if created:
        roles = Role.objects.filter(
            structure=structure,
        )
        for role in roles:
            try:
                person = Person.objects.get(
                    bhs_pk=role.human.id,
                )
            except Person.DoesNotExist as e:
                log.error("{0} {1}".format(e, person))
                continue
            status = Member.STATUS.active
            is_admin = True
            defaults = {
                'status': status,
                'is_admin': is_admin,
            }
            try:
                member, created = Member.objects.update_or_create(
                    person=person,
                    group=group,
                    defaults=defaults,
                )
            except IntegrityError as e:
                log.error("{0} {1}".format(e, member))
                return
        try:
            parent_bhs_id = structure.parent.bhs_id
        except AttributeError:
            return
        if parent_bhs_id:
            try:
                organization = Organization.objects.get(
                    bhs_id=parent_bhs_id,
                )
            except Organization.DoesNotExist:
                organization = None
            group.organization = organization
            group.save()

def update_or_create_member_from_smjoin(smjoin):
    if smjoin.structure.kind == 'district':
        # Ignore districts
        log.error("District mismatch")
        return
    elif smjoin.structure.kind == 'organization':
        # Must abstract this because we can't trust updated_ts
        human = smjoin.subscription.human
        smjoin = SMJoin.objects.filter(
            structure=smjoin.structure,
            subscription__human=human,
        ).order_by('established_date').last()
        # Extract current Subscription from SMJoin
        subscription = smjoin.subscription
        try:
            person = Person.objects.get(
                bhs_pk=human.id,
            )
        except Person.DoesNotExist as e:
            log.error("{0} {1}".format(e, human))
            return
        if person.email:
            if subscription.status == 'active':
                status = Person.STATUS.active
                is_valid = True
            else:
                status = Person.STATUS.inactive
                is_valid = False
        else:
            if subscription.status == 'active':
                status = Person.STATUS.missing
                is_valid = True
            else:
                status = Person.STATUS.legacy
                is_valid = False
        valid_through = subscription.valid_through
        person.status = status
        person.is_valid = is_valid
        person.valid_through = valid_through
        try:
            person.full_clean()
        except ValidationError as e:
            log.error("{0} {1}".format(e, person))
            return
        person.save()
        if status == Person.STATUS.active:
            user.is_active = True
            user.save()
        else:
            user.is_active = False
            user.save()
        log.info("{0} {1}".format(person, valid_through))
        return
    elif smjoin.structure.kind == 'chapter':
        # Must abstract this because we can't trust updated_ts
        smjoin = SMJoin.objects.filter(
            structure=smjoin.structure,
            subscription__human=smjoin.subscription.human,
        ).order_by('established_date').last()
        if smjoin.subscription.status == 'active':
            status = 10
        else:
            status = -10
        try:
            group = Group.objects.get(
                bhs_pk=smjoin.structure.id
            )
        except Group.DoesNotExist as e:
            # Probably due to pending status
            log.error("{0} {1}".format(e, smjoin))
            return
        try:
            person = Person.objects.get(
                bhs_pk=smjoin.subscription.human.id
            )
        except Person.DoesNotExist as e:
            # Generally an error
            log.error("{0} {1}".format(e, smjoin))
            return
        defaults = {
            'status': status,
        }
        try:
            member, created = Member.objects.update_or_create(
                person=person,
                group=group,
                defaults=defaults,
            )
            log.info("{0}; {1}".format(member, created))
        except IntegrityError as e:
            log.error("{0} {1}".format(e, smjoin))
            return
    elif smjoin.structure.kind == 'quartet':
        # Assumes quartets have precisely one Subscription
        smjoin = smjoin.subscription.smjoins.order_by('established_date').last()
        try:
            part_stripped = smjoin.vocal_part.strip()
        except AttributeError:
            part_stripped = "Unknown"
        if part_stripped.casefold() == 'Tenor'.casefold():
            part = 1
        elif part_stripped.casefold() == 'Lead'.casefold():
            part = 2
        elif part_stripped.casefold() == 'Baritone'.casefold():
            part = 3
        elif part_stripped.casefold() == 'Bass'.casefold():
            part = 4
        else:
            part = None
        if smjoin.subscription.status == 'active':
            status = 10
        else:
            status = -10
        try:
            group = Group.objects.get(
                bhs_pk=smjoin.structure.id
            )
        except Group.DoesNotExist as e:
            # Probably due to pending status
            log.error("{0} {1}".format(e, smjoin))
            return
        try:
            person = Person.objects.get(
                bhs_pk=smjoin.subscription.human.id
            )
        except Person.DoesNotExist as e:
            # Generally an error
            log.error("{0} {1}".format(e, smjoin))
            return
        defaults = {
            'status': status,
            'part': part,
        }
        try:
            member, created = Member.objects.update_or_create(
                person=person,
                group=group,
                defaults=defaults,
            )
            log.info("{0}; {1}".format(member, created))
        except IntegrityError as e:
            log.error("{0} {1}".format(e, smjoin))
            return
    else:
        # This is an error.
        log.error("Unknown Kind")
        return
<commit_msg>Fix traverse<commit_after># Standard Libary
import logging

from email_validator import (
    EmailNotValidError,
    validate_email,
)

# Django
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.utils import encoding

# First-Party
from api.models import (
    Group,
    Member,
    Organization,
    Person,
    User,
)
# Remote
from bhs.models import (
    Role,
    SMJoin,
)

log = logging.getLogger('updater')

def update_or_create_person_from_human(human):
    first_name = human.first_name.strip()
    middle_name = human.middle_name.strip()
    last_name = human.last_name.strip()
    nick_name = human.nick_name.replace("'", "").replace('"', '').replace("(", "").replace(")", "").strip()
    if not first_name:
        first_name = None
    if not middle_name:
        middle_name = None
    if not last_name:
        last_name = None
    if nick_name and (nick_name != first_name):
        nick_name = "({0})".format(nick_name)
    else:
        nick_name = None
    name = " ".join(
        map(
            (lambda x: encoding.smart_text(x)),
            filter(
                None, [
                    first_name,
                    middle_name,
                    last_name,
                    nick_name,
                ]
            )
        )
    )
    try:
        v = validate_email(human.email.strip())
        email = v["email"].lower()
    except EmailNotValidError as e:
        email = None
    birth_date = human.birth_date
    if human.phone:
        phone = human.phone
    else:
        phone = ''
    if human.cell_phone:
        cell_phone = human.cell_phone
    else:
        cell_phone = ''
    if human.work_phone:
        work_phone = human.work_phone
    else:
        work_phone = ''
    is_bhs = True
    bhs_id = human.bhs_id
    if human.sex:
        if human.sex.casefold() == 'male'.casefold():
            gender = 10
        elif human.sex.casefold() == 'female'.casefold():
            gender = 20
        else:
            gender = None
    else:
        gender = None
    if human.primary_voice_part:
        if human.primary_voice_part.casefold() == 'Tenor'.casefold():
            part = 1
        elif human.primary_voice_part.casefold() == 'Lead'.casefold():
            part = 2
        elif human.primary_voice_part.casefold() == 'Baritone'.casefold():
            part = 3
        elif human.primary_voice_part.casefold() == 'Bass'.casefold():
            part = 4
        else:
            part = None
    else:
        part = None
    defaults = {
        'name': name,
        'email': email,
        'birth_date': birth_date,
        'phone': phone,
        'cell_phone': cell_phone,
        'work_phone': work_phone,
        'is_bhs': is_bhs,
        'bhs_id': bhs_id,
        'gender': gender,
        'part': part,
    }
    try:
        person, created = Person.objects.update_or_create(
            bhs_pk=human.id,
            defaults=defaults,
        )
        if created and email:
            User.objects.create_user(
                person=person,
                is_active=False,
            )
        log.info("{0}; {1}".format(person, created))
    except IntegrityError as e:
        log.error("{0} {1}".format(e, human))
        return

def update_or_create_group_from_structure(structure):
    kind_map = {
        'chapter': 32,
        'quartet': 41,
    }
    try:
        kind = kind_map[structure.kind]
    except KeyError as e:
        log.error("{0} {1}".format(e, structure))
        return
    status_map = {
        'active': 10,
        'active-licensed': 10,
        'pending': 0,
        'pending-voluntary': 0,
        'expired': -10,
        'closed-revoked': -10,
        'closed-merged': -10,
        'suspended-membership': -10,
        'cancelled': -10,
        'lapsed': -10,
        'closed-voluntary': -10,
        'expelled': -10,
        'suspended': -10,
    }
    if kind == 41:
        if structure.name:
            name = structure.name.strip()
        else:
            if not structure.preferred_name:
                return
            name = "{0} (NAME APPROVAL PENDING)".format(
                structure.preferred_name.strip()
            )
    else:
        name = "{0} [{1}]".format(
            structure.chorus_name.strip(),
            structure.name.strip(),
        )
    status = status_map[str(structure.status)]
    if kind == 32:
        code = structure.chapter_code
    else:
        code = ''
    start_date = structure.established_date
    try:
        v = validate_email(structure.email)
        email = v["email"].lower()
    except EmailNotValidError:
        email = ''
    if structure.phone:
        phone = structure.phone
    else:
        phone = ''
    is_bhs = True
    bhs_id = structure.bhs_id
    defaults = {
        'name': name,
        'status': status,
        'kind': kind,
        'code': code,
        'start_date': start_date,
        'email': email,
        'phone': phone,
        'is_bhs': is_bhs,
        'bhs_id': bhs_id,
    }
    try:
        group, created = Group.objects.update_or_create(
            bhs_pk=structure.id,
            defaults=defaults,
        )
        log.info("{0}; {1}".format(group, created))
    except IntegrityError as e:
        log.error("{0} {1}".format(e, group))
        return
    if created:
        roles = Role.objects.filter(
            structure=structure,
        )
        for role in roles:
            try:
                person = Person.objects.get(
                    bhs_pk=role.human.id,
                )
            except Person.DoesNotExist as e:
                log.error("{0} {1}".format(e, person))
                continue
            status = Member.STATUS.active
            is_admin = True
            defaults = {
                'status': status,
                'is_admin': is_admin,
            }
            try:
                member, created = Member.objects.update_or_create(
                    person=person,
                    group=group,
                    defaults=defaults,
                )
            except IntegrityError as e:
                log.error("{0} {1}".format(e, member))
                return
        try:
            parent_bhs_id = structure.parent.bhs_id
        except AttributeError:
            return
        if parent_bhs_id:
            try:
                organization = Organization.objects.get(
                    bhs_id=parent_bhs_id,
                )
            except Organization.DoesNotExist:
                organization = None
            group.organization = organization
            group.save()

def update_or_create_member_from_smjoin(smjoin):
    if smjoin.structure.kind == 'district':
        # Ignore districts
        log.error("District mismatch")
        return
    elif smjoin.structure.kind == 'organization':
        # Must abstract this because we can't trust updated_ts
        human = smjoin.subscription.human
        smjoin = SMJoin.objects.filter(
            structure=smjoin.structure,
            subscription__human=human,
        ).order_by('established_date').last()
        # Extract current Subscription from SMJoin
        subscription = smjoin.subscription
        try:
            person = Person.objects.get(
                bhs_pk=human.id,
            )
        except Person.DoesNotExist as e:
            log.error("{0} {1}".format(e, human))
            return
        if person.email:
            if subscription.status == 'active':
                status = Person.STATUS.active
                is_valid = True
            else:
                status = Person.STATUS.inactive
                is_valid = False
        else:
            if subscription.status == 'active':
                status = Person.STATUS.missing
                is_valid = True
            else:
                status = Person.STATUS.legacy
                is_valid = False
        valid_through = subscription.valid_through
        person.status = status
        person.is_valid = is_valid
        person.valid_through = valid_through
        try:
            person.full_clean()
        except ValidationError as e:
            log.error("{0} {1}".format(e, person))
            return
        person.save()
        if status == Person.STATUS.active:
            person.user.is_active = True
            person.user.save()
        else:
            person.user.is_active = False
            person.user.save()
        log.info("{0} {1}".format(person, valid_through))
        return
    elif smjoin.structure.kind == 'chapter':
        # Must abstract this because we can't trust updated_ts
        smjoin = SMJoin.objects.filter(
            structure=smjoin.structure,
            subscription__human=smjoin.subscription.human,
        ).order_by('established_date').last()
        if smjoin.subscription.status == 'active':
            status = 10
        else:
            status = -10
        try:
            group = Group.objects.get(
                bhs_pk=smjoin.structure.id
            )
        except Group.DoesNotExist as e:
            # Probably due to pending status
            log.error("{0} {1}".format(e, smjoin))
            return
        try:
            person = Person.objects.get(
                bhs_pk=smjoin.subscription.human.id
            )
        except Person.DoesNotExist as e:
            # Generally an error
            log.error("{0} {1}".format(e, smjoin))
            return
        defaults = {
            'status': status,
        }
        try:
            member, created = Member.objects.update_or_create(
                person=person,
                group=group,
                defaults=defaults,
            )
            log.info("{0}; {1}".format(member, created))
        except IntegrityError as e:
            log.error("{0} {1}".format(e, smjoin))
            return
    elif smjoin.structure.kind == 'quartet':
        # Assumes quartets have precisely one Subscription
        smjoin = smjoin.subscription.smjoins.order_by('established_date').last()
        try:
            part_stripped = smjoin.vocal_part.strip()
        except AttributeError:
            part_stripped = "Unknown"
        if part_stripped.casefold() == 'Tenor'.casefold():
            part = 1
        elif part_stripped.casefold() == 'Lead'.casefold():
            part = 2
        elif part_stripped.casefold() == 'Baritone'.casefold():
            part = 3
        elif part_stripped.casefold() == 'Bass'.casefold():
            part = 4
        else:
            part = None
        if smjoin.subscription.status == 'active':
            status = 10
        else:
            status = -10
        try:
            group = Group.objects.get(
                bhs_pk=smjoin.structure.id
            )
        except Group.DoesNotExist as e:
            # Probably due to pending status
            log.error("{0} {1}".format(e, smjoin))
            return
        try:
            person = Person.objects.get(
                bhs_pk=smjoin.subscription.human.id
            )
        except Person.DoesNotExist as e:
            # Generally an error
            log.error("{0} {1}".format(e, smjoin))
            return
        defaults = {
            'status': status,
            'part': part,
        }
        try:
            member, created = Member.objects.update_or_create(
                person=person,
                group=group,
                defaults=defaults,
            )
            log.info("{0}; {1}".format(member, created))
        except IntegrityError as e:
            log.error("{0} {1}".format(e, smjoin))
            return
    else:
        # This is an error.
        log.error("Unknown Kind")
        return
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import json
import time
import requests
from bs4 import BeautifulSoup

def get_dilbert_comic(date="2016-01-01"):
    """
    Fetches Dilbert comic's image URL & topic for the given date
    (Future dates get redirected to latest date's comic, automatically)
    Input -
        date : YYYY-MM-DD format
    Output -
        {
            success : True/False,
            date    : string,
            url     : string,
            title   : string,
            image   : string,
        }
    """
    url = "http://dilbert.com/strip/{0}".format(date)
    comic = {
        'success': False,
        'date': date,
        'url': url,
        'title': '',
        'image': '',
    }

    response = requests.get(url)
    if response.status_code == 200:
        soup = BeautifulSoup(response.content, "html.parser")
        try:
            # title is empty at times (e.g. 2016-03-20 comic)
            comic['title'] = soup.find(attrs={"property":"og:title"})['content']
            comic['image'] = soup.find(attrs={"property":"og:image"})['content']
            comic['success'] = True
        except (AttributeError, KeyError):
            pass
    return comic

if __name__ == '__main__':
    comic = get_dilbert_comic(date=time.strftime('%Y-%m-%d'))
<commit_msg>+Quote of the Day fetcher<commit_after>#!/usr/bin/python
import os
import json
import time
import requests
from bs4 import BeautifulSoup

def get_dilbert_comic(date="2016-01-01"):
    """
    Fetches Dilbert comic's image URL & topic for the given date
    (Future dates get redirected to latest date's comic, automatically)
    Input -
        date : YYYY-MM-DD format
    Output -
        {
            success : True/False,
            date    : string,
            url     : string,
            title   : string,
            image   : string,
        }
    """
    url = "http://dilbert.com/strip/{0}".format(date)
    comic = {
        'success': False,
        'date': date,
        'url': url,
        'title': '',
        'image': '',
    }

    response = requests.get(url)
    if response.status_code == 200:
        soup = BeautifulSoup(response.content, "html.parser")
        try:
            # title is empty at times (e.g. 2016-03-20 comic)
            comic['title'] = soup.find(attrs={"property":"og:title"})['content']
            comic['image'] = soup.find(attrs={"property":"og:image"})['content']
            comic['success'] = True
        except (AttributeError, KeyError):
            pass
    return comic

def get_QOTD():
    """
    Fetches QOTD from the REST API of Quotes.rest
    Output -
        {
            success    : True/False,
            date       : string,
            quote      : string,
            author     : string,
            background : string,
        }
    """
    qotd = {
        'success' : False,
        'date' : '',
        'quote' : '',
        'author' : '',
        'background' : '',
    }
    response = requests.get("http://quotes.rest/qod.json")
    print response.status_code, response.content
    if response.status_code == 200:
        try:
            json_response = response.json()['contents']['quotes'][0]
            for detail in ['date', 'quote', 'author', 'background']:
                qotd[detail] = json_response[detail].strip()
            qotd['success'] = True
        except (AttributeError, KeyError, IndexError):
            pass
    return qotd

if __name__ == '__main__':
    comic = get_dilbert_comic(date=time.strftime('%Y-%m-%d'))
    qotd  = get_QOTD()
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import uuid

import httpretty
from six.moves.urllib import parse as urlparse

from keystoneclient.openstack.common import jsonutils
from keystoneclient.tests import utils
from keystoneclient.v3 import client

TestResponse = utils.TestResponse

def parameterize(ref):
    """Rewrites attributes to match the kwarg naming convention in client.

    >>> parameterize({'project_id': 0})
    {'project': 0}

    """
    params = ref.copy()
    for key in ref:
        if key[-3:] == '_id':
            params.setdefault(key[:-3], params.pop(key))
    return params

class UnauthenticatedTestCase(utils.TestCase):
    """Class used as base for unauthenticated calls."""

    TEST_ROOT_URL = 'http://127.0.0.1:5000/'
    TEST_URL = '%s%s' % (TEST_ROOT_URL, 'v3')
    TEST_ROOT_ADMIN_URL = 'http://127.0.0.1:35357/'
    TEST_ADMIN_URL = '%s%s' % (TEST_ROOT_ADMIN_URL, 'v3')

class TestCase(UnauthenticatedTestCase):

    TEST_SERVICE_CATALOG = [{
        "endpoints": [{
            "url": "http://cdn.admin-nets.local:8774/v1.0/",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://127.0.0.1:8774/v1.0",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://cdn.admin-nets.local:8774/v1.0",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "nova_compat"
    }, {
        "endpoints": [{
            "url": "http://nova/novapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://nova/novapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://nova/novapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "compute"
    }, {
        "endpoints": [{
            "url": "http://glance/glanceapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://glance/glanceapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://glance/glanceapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "image",
        "name": "glance"
    }, {
        "endpoints": [{
            "url": "http://127.0.0.1:5000/v3",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://127.0.0.1:5000/v3",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://127.0.0.1:35357/v3",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "identity"
    }, {
        "endpoints": [{
            "url": "http://swift/swiftapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://swift/swiftapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://swift/swiftapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "object-store"
    }]

    def setUp(self):
        super(TestCase, self).setUp()
        self.client = client.Client(username=self.TEST_USER,
                                    token=self.TEST_TOKEN,
                                    tenant_name=self.TEST_TENANT_NAME,
                                    auth_url=self.TEST_URL,
                                    endpoint=self.TEST_URL)

    def stub_auth(self, subject_token=None, **kwargs):
        if not subject_token:
            subject_token = self.TEST_TOKEN

        self.stub_url(httpretty.POST, ['auth', 'tokens'],
                      X_Subject_Token=subject_token, **kwargs)

class CrudTests(object):
    key = None
    collection_key = None
    model = None
    manager = None
    path_prefix = None

    def new_ref(self, **kwargs):
        kwargs.setdefault('id', uuid.uuid4().hex)
        return kwargs

    def encode(self, entity):
        if isinstance(entity, dict):
            return {self.key: entity}
        if isinstance(entity, list):
            return {self.collection_key: entity}
        raise NotImplementedError('Are you sure you want to encode that?')

    def stub_entity(self, method, parts=None, entity=None, id=None, **kwargs):
        if entity:
            entity = self.encode(entity)
            kwargs['json'] = entity

        if not parts:
            parts = [self.collection_key]

            if self.path_prefix:
                parts.insert(0, self.path_prefix)

        if id:
            if not parts:
                parts = []

            parts.append(id)

        self.stub_url(method, parts=parts, **kwargs)

    def assertEntityRequestBodyIs(self, entity):
        self.assertRequestBodyIs(json=self.encode(entity))

    @httpretty.activate
    def test_create(self, ref=None, req_ref=None):
        ref = ref or self.new_ref()
        manager_ref = ref.copy()
        manager_ref.pop('id')

        # req_ref argument allows you to specify a different
        # signature for the request when the manager does some
        # conversion before doing the request (e.g converting
        # from datetime object to timestamp string)
        req_ref = req_ref or ref.copy()
        req_ref.pop('id')

        self.stub_entity(httpretty.POST, entity=req_ref, status=201)

        returned = self.manager.create(**parameterize(manager_ref))
        self.assertTrue(isinstance(returned, self.model))
        for attr in req_ref:
            self.assertEqual(
                getattr(returned, attr),
                req_ref[attr],
                'Expected different %s' % attr)
        self.assertEntityRequestBodyIs(req_ref)

    @httpretty.activate
    def test_get(self, ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.GET, id=ref['id'], entity=ref)

        returned = self.manager.get(ref['id'])
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)

    @httpretty.activate
    def test_list(self, ref_list=None, expected_path=None, **filter_kwargs):
        ref_list = ref_list or [self.new_ref(), self.new_ref()]

        if not expected_path:
            if self.path_prefix:
                expected_path = 'v3/%s/%s' % (self.path_prefix,
                                              self.collection_key)
            else:
                expected_path = 'v3/%s' % self.collection_key

        httpretty.register_uri(httpretty.GET,
                               urlparse.urljoin(self.TEST_URL, expected_path),
                               body=jsonutils.dumps(self.encode(ref_list)))

        returned_list = self.manager.list(**filter_kwargs)
        self.assertTrue(len(returned_list))
        [self.assertTrue(isinstance(r, self.model)) for r in returned_list]

    @httpretty.activate
    def test_find(self, ref=None):
        ref = ref or self.new_ref()
        ref_list = [ref]

        self.stub_entity(httpretty.GET, entity=ref_list)

        returned = self.manager.find(name=getattr(ref, 'name', None))
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)

        if hasattr(ref, 'name'):
            self.assertQueryStringIs({'name': ref['name']})
        else:
            self.assertQueryStringIs({})

    @httpretty.activate
    def test_update(self, ref=None, req_ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.PATCH, id=ref['id'], entity=ref)

        # req_ref argument allows you to specify a different
        # signature for the request when the manager does some
        # conversion before doing the request (e.g converting
        # from datetime object to timestamp string)
        req_ref = (req_ref or ref).copy()
        req_ref.pop('id')

        returned = self.manager.update(ref['id'], **parameterize(req_ref))
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)
        self.assertEntityRequestBodyIs(req_ref)

    @httpretty.activate
    def test_delete(self, ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.DELETE, id=ref['id'], status=204)
        self.manager.delete(ref['id'])
<commit_msg>v3 test utils, don't modify input parameter<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import uuid

import httpretty
from six.moves.urllib import parse as urlparse

from keystoneclient.openstack.common import jsonutils
from keystoneclient.tests import utils
from keystoneclient.v3 import client

TestResponse = utils.TestResponse

def parameterize(ref):
    """Rewrites attributes to match the kwarg naming convention in client.

    >>> parameterize({'project_id': 0})
    {'project': 0}

    """
    params = ref.copy()
    for key in ref:
        if key[-3:] == '_id':
            params.setdefault(key[:-3], params.pop(key))
    return params

class UnauthenticatedTestCase(utils.TestCase):
    """Class used as base for unauthenticated calls."""

    TEST_ROOT_URL = 'http://127.0.0.1:5000/'
    TEST_URL = '%s%s' % (TEST_ROOT_URL, 'v3')
    TEST_ROOT_ADMIN_URL = 'http://127.0.0.1:35357/'
    TEST_ADMIN_URL = '%s%s' % (TEST_ROOT_ADMIN_URL, 'v3')

class TestCase(UnauthenticatedTestCase):

    TEST_SERVICE_CATALOG = [{
        "endpoints": [{
            "url": "http://cdn.admin-nets.local:8774/v1.0/",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://127.0.0.1:8774/v1.0",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://cdn.admin-nets.local:8774/v1.0",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "nova_compat"
    }, {
        "endpoints": [{
            "url": "http://nova/novapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://nova/novapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://nova/novapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "compute"
    }, {
        "endpoints": [{
            "url": "http://glance/glanceapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://glance/glanceapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://glance/glanceapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "image",
        "name": "glance"
    }, {
        "endpoints": [{
            "url": "http://127.0.0.1:5000/v3",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://127.0.0.1:5000/v3",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://127.0.0.1:35357/v3",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "identity"
    }, {
        "endpoints": [{
            "url": "http://swift/swiftapi/public",
            "region": "RegionOne",
            "interface": "public"
        }, {
            "url": "http://swift/swiftapi/internal",
            "region": "RegionOne",
            "interface": "internal"
        }, {
            "url": "http://swift/swiftapi/admin",
            "region": "RegionOne",
            "interface": "admin"
        }],
        "type": "object-store"
    }]

    def setUp(self):
        super(TestCase, self).setUp()
        self.client = client.Client(username=self.TEST_USER,
                                    token=self.TEST_TOKEN,
                                    tenant_name=self.TEST_TENANT_NAME,
                                    auth_url=self.TEST_URL,
                                    endpoint=self.TEST_URL)

    def stub_auth(self, subject_token=None, **kwargs):
        if not subject_token:
            subject_token = self.TEST_TOKEN

        self.stub_url(httpretty.POST, ['auth', 'tokens'],
                      X_Subject_Token=subject_token, **kwargs)

class CrudTests(object):
    key = None
    collection_key = None
    model = None
    manager = None
    path_prefix = None

    def new_ref(self, **kwargs):
        kwargs.setdefault('id', uuid.uuid4().hex)
        return kwargs

    def encode(self, entity):
        if isinstance(entity, dict):
            return {self.key: entity}
        if isinstance(entity, list):
            return {self.collection_key: entity}
        raise NotImplementedError('Are you sure you want to encode that?')

    def stub_entity(self, method, parts=None, entity=None, id=None, **kwargs):
        if entity:
            entity = self.encode(entity)
            kwargs['json'] = entity

        if not parts:
            parts = [self.collection_key]

            if self.path_prefix:
                parts.insert(0, self.path_prefix)

        if id:
            if not parts:
                parts = []

            parts.append(id)

        self.stub_url(method, parts=parts, **kwargs)

    def assertEntityRequestBodyIs(self, entity):
        self.assertRequestBodyIs(json=self.encode(entity))

    @httpretty.activate
    def test_create(self, ref=None, req_ref=None):
        ref = ref or self.new_ref()
        manager_ref = ref.copy()
        manager_ref.pop('id')

        # req_ref argument allows you to specify a different
        # signature for the request when the manager does some
        # conversion before doing the request (e.g converting
        # from datetime object to timestamp string)
        req_ref = (req_ref or ref).copy()
        req_ref.pop('id')

        self.stub_entity(httpretty.POST, entity=req_ref, status=201)

        returned = self.manager.create(**parameterize(manager_ref))
        self.assertTrue(isinstance(returned, self.model))
        for attr in req_ref:
            self.assertEqual(
                getattr(returned, attr),
                req_ref[attr],
                'Expected different %s' % attr)
        self.assertEntityRequestBodyIs(req_ref)

    @httpretty.activate
    def test_get(self, ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.GET, id=ref['id'], entity=ref)

        returned = self.manager.get(ref['id'])
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)

    @httpretty.activate
    def test_list(self, ref_list=None, expected_path=None, **filter_kwargs):
        ref_list = ref_list or [self.new_ref(), self.new_ref()]

        if not expected_path:
            if self.path_prefix:
                expected_path = 'v3/%s/%s' % (self.path_prefix,
                                              self.collection_key)
            else:
                expected_path = 'v3/%s' % self.collection_key

        httpretty.register_uri(httpretty.GET,
                               urlparse.urljoin(self.TEST_URL, expected_path),
                               body=jsonutils.dumps(self.encode(ref_list)))

        returned_list = self.manager.list(**filter_kwargs)
        self.assertTrue(len(returned_list))
        [self.assertTrue(isinstance(r, self.model)) for r in returned_list]

    @httpretty.activate
    def test_find(self, ref=None):
        ref = ref or self.new_ref()
        ref_list = [ref]

        self.stub_entity(httpretty.GET, entity=ref_list)

        returned = self.manager.find(name=getattr(ref, 'name', None))
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)

        if hasattr(ref, 'name'):
            self.assertQueryStringIs({'name': ref['name']})
        else:
            self.assertQueryStringIs({})

    @httpretty.activate
    def test_update(self, ref=None, req_ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.PATCH, id=ref['id'], entity=ref)

        # req_ref argument allows you to specify a different
        # signature for the request when the manager does some
        # conversion before doing the request (e.g converting
        # from datetime object to timestamp string)
        req_ref = (req_ref or ref).copy()
        req_ref.pop('id')

        returned = self.manager.update(ref['id'], **parameterize(req_ref))
        self.assertTrue(isinstance(returned, self.model))
        for attr in ref:
            self.assertEqual(
                getattr(returned, attr),
                ref[attr],
                'Expected different %s' % attr)
        self.assertEntityRequestBodyIs(req_ref)

    @httpretty.activate
    def test_delete(self, ref=None):
        ref = ref or self.new_ref()

        self.stub_entity(httpretty.DELETE, id=ref['id'], status=204)
        self.manager.delete(ref['id'])
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''

Usage:
    validate.py [path]

Validates all xml files against the DocBook 5 RELAX NG schema, and
attempts to build all books.

Options:
    path     Root directory, defaults to <repo root>/doc/src/doc/docbkx

Ignores pom.xml files and subdirectories named "target".

Requires:
    - Python 2.7 or greater (for argparse)
    - lxml Python library
    - Maven

'''
from lxml import etree

import argparse
import os
import subprocess
import sys
import urllib2

# These are files that are known to not be in DocBook format
FILE_EXCEPTIONS = ['ha-guide-docinfo.xml']

# These are books that we aren't checking yet
BOOK_EXCEPTIONS = ['openstack-training']

def get_schema():
    """Return the DocBook RELAX NG schema"""
    url = "http://www.oasis-open.org/docbook/xml/5.0b5/rng/docbookxi.rng"
    relaxng_doc = etree.parse(urllib2.urlopen(url))
    return etree.RelaxNG(relaxng_doc)

def validation_failed(schema, doc):
    """Return True if the parsed doc fails against the schema

    This will ignore validation failures of the type: IDREF attribute linkend
    references an unknown ID. This is because we are validating individual
    files that are being imported, and sometimes the reference isn't present
    in the current file."""
    return not schema.validate(doc) and \
        any(log.type_name != "DTD_UNKNOWN_ID" for log in schema.error_log)

def verify_section_tags_have_xmid(doc):
    """Check that all section tags have an xml:id attribute

    Will throw an exception if there's at least one missing"""
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    for node in doc.xpath('//docbook:section', namespaces=ns):
        if "{http://www.w3.org/XML/1998/namespace}id" not in node.attrib:
            raise ValueError("section missing xml:id attribute, line %d" %
                             node.sourceline)

def error_message(error_log):
    """Return a string that contains the error message.

    We use this to filter out false positives related to IDREF attributes
    """
    errs = [str(x) for x in error_log if x.type_name != 'DTD_UNKNOWN_ID']

    # Reverse output so that earliest failures are reported first
    errs.reverse()
    return "\n".join(errs)

def validate_individual_files(rootdir, exceptions):
    schema = get_schema()

    any_failures = False

    for root, dirs, files in os.walk(rootdir):
        # Don't descend into 'target' subdirectories
        try:
            ind = dirs.index('target')
            del dirs[ind]
        except ValueError:
            pass

        for f in files:
            # Ignore maven files, which are called pom.xml
            if f.endswith('.xml') and f != 'pom.xml' \
                                  and f not in exceptions:
                try:
                    path = os.path.abspath(os.path.join(root, f))
                    doc = etree.parse(path)
                    if validation_failed(schema, doc):
                        any_failures = True
                        print error_message(schema.error_log)
                    verify_section_tags_have_xmid(doc)
                except etree.XMLSyntaxError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)
                except ValueError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)

        if any_failures:
            sys.exit(1)

def build_all_books(rootdir, exceptions):
    """ Build all of the books.

    Looks for all directories with "pom.xml" in them and runs
    "mvn clean generate-sources" in that directory.

    This will throw an exception if a book fails to build
    """
    for root, dirs, files in os.walk(rootdir):
        book = os.path.basename(root)
        if ("pom.xml" in files) and (book not in exceptions):
            print "Building %s" % book
            os.chdir(root)
            subprocess.check_call(["mvn", "clean", "generate-sources"])

def main(rootdir):
    validate_individual_files(rootdir, FILE_EXCEPTIONS)
    build_all_books(rootdir, BOOK_EXCEPTIONS)

def default_root():
    """Return the location of openstack-manuals/doc/src/docbkx

    The current working directory must be inside of the openstack-manuals
    repository for this method to succeed"""
    args = ["git", "rev-parse", "--show-toplevel"]
    gitroot = subprocess.check_output(args).rstrip()
    return os.path.join(gitroot, "doc/src/docbkx")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Validate XML files against "
                                     "the DocBook 5 RELAX NG schema")
    parser.add_argument('path', nargs='?', default=default_root(),
                        help="Root directory that contains DocBook files, "
                        "defaults to `git rev-parse --show-toplevel`/doc/src/"
                        "docbkx")
    args = parser.parse_args()
    main(args.path)
<commit_msg>do not exit after first invalid file<commit_after>#!/usr/bin/env python
'''

Usage:
    validate.py [path]

Validates all xml files against the DocBook 5 RELAX NG schema, and
attempts to build all books.

Options:
    path     Root directory, defaults to <repo root>/doc/src/doc/docbkx

Ignores pom.xml files and subdirectories named "target".

Requires:
    - Python 2.7 or greater (for argparse)
    - lxml Python library
    - Maven

'''
from lxml import etree

import argparse
import os
import subprocess
import sys
import urllib2

# These are files that are known to not be in DocBook format
FILE_EXCEPTIONS = ['ha-guide-docinfo.xml']

# These are books that we aren't checking yet
BOOK_EXCEPTIONS = ['openstack-training']

def get_schema():
    """Return the DocBook RELAX NG schema"""
    url = "http://www.oasis-open.org/docbook/xml/5.0b5/rng/docbookxi.rng"
    relaxng_doc = etree.parse(urllib2.urlopen(url))
    return etree.RelaxNG(relaxng_doc)

def validation_failed(schema, doc):
    """Return True if the parsed doc fails against the schema

    This will ignore validation failures of the type: IDREF attribute linkend
    references an unknown ID. This is because we are validating individual
    files that are being imported, and sometimes the reference isn't present
    in the current file."""
    return not schema.validate(doc) and \
        any(log.type_name != "DTD_UNKNOWN_ID" for log in schema.error_log)

def verify_section_tags_have_xmid(doc):
    """Check that all section tags have an xml:id attribute

    Will throw an exception if there's at least one missing"""
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    for node in doc.xpath('//docbook:section', namespaces=ns):
        if "{http://www.w3.org/XML/1998/namespace}id" not in node.attrib:
            raise ValueError("section missing xml:id attribute, line %d" %
                             node.sourceline)

def error_message(error_log):
    """Return a string that contains the error message.

    We use this to filter out false positives related to IDREF attributes
    """
    errs = [str(x) for x in error_log if x.type_name != 'DTD_UNKNOWN_ID']

    # Reverse output so that earliest failures are reported first
    errs.reverse()
    return "\n".join(errs)

def validate_individual_files(rootdir, exceptions):
    schema = get_schema()

    any_failures = False

    for root, dirs, files in os.walk(rootdir):
        # Don't descend into 'target' subdirectories
        try:
            ind = dirs.index('target')
            del dirs[ind]
        except ValueError:
            pass

        for f in files:
            # Ignore maven files, which are called pom.xml
            if f.endswith('.xml') and f != 'pom.xml' \
                                  and f not in exceptions:
                try:
                    path = os.path.abspath(os.path.join(root, f))
                    doc = etree.parse(path)
                    if validation_failed(schema, doc):
                        any_failures = True
                        print error_message(schema.error_log)
                    verify_section_tags_have_xmid(doc)
                except etree.XMLSyntaxError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)
                except ValueError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)

    if any_failures:
        sys.exit(1)

def build_all_books(rootdir, exceptions):
    """ Build all of the books.

    Looks for all directories with "pom.xml" in them and runs
    "mvn clean generate-sources" in that directory.

    This will throw an exception if a book fails to build
    """
    for root, dirs, files in os.walk(rootdir):
        book = os.path.basename(root)
        if ("pom.xml" in files) and (book not in exceptions):
            print "Building %s" % book
            os.chdir(root)
            subprocess.check_call(["mvn", "clean", "generate-sources"])

def main(rootdir):
    validate_individual_files(rootdir, FILE_EXCEPTIONS)
    build_all_books(rootdir, BOOK_EXCEPTIONS)

def default_root():
    """Return the location of openstack-manuals/doc/src/docbkx

    The current working directory must be inside of the openstack-manuals
    repository for this method to succeed"""
    args = ["git", "rev-parse", "--show-toplevel"]
    gitroot = subprocess.check_output(args).rstrip()
    return os.path.join(gitroot, "doc/src/docbkx")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Validate XML files against "
                                     "the DocBook 5 RELAX NG schema")
    parser.add_argument('path', nargs='?', default=default_root(),
                        help="Root directory that contains DocBook files, "
                        "defaults to `git rev-parse --show-toplevel`/doc/src/"
                        "docbkx")
    args = parser.parse_args()
    main(args.path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2015 Vincent Jacques <vincent@vincent-jacques.net>

"""
Define a variadic function:

    >>> @variadic(int)
    ... def f(*xs):
    ...   return xs

It can be called with a variable number of arguments,
and they are passed to decorated function as a single, iterable, parameter:

    >>> f()
    ()
    >>> f(1)
    (1,)
    >>> f(1, 2, 3, 4)
    (1, 2, 3, 4)

But it can also be called with lists (any iterable, in fact) of arguments:

    >>> f([])
    ()
    >>> f([1, 2, 3], [4, 5, 6])
    (1, 2, 3, 4, 5, 6)
    >>> f(xrange(1, 4))
    (1, 2, 3)

And you can even mix them:

    >>> f(1, [2, 3], 4, xrange(5, 8))
    (1, 2, 3, 4, 5, 6, 7)
"""

import ast
import functools
import inspect
import itertools
import sys
import types
import unittest

# Todo-list:
# - allow passing the flatten function instead of typ
# - allow an even simpler usage without any parameters
# - add a parameter string to be prepended or appended to the docstring
# - support decorating callables that are not functions?
# - add a doctest showing stack trace when decorated function raises
# - autodoc a decorated function

# >>> help(types.FunctionType)
# class function(object)
#  |  function(code, globals[, name[, argdefs[, closure]]])
#  |
#  |  Create a function object from a code object and a dictionary.
#  |  The optional name string overrides the name from the code object.
#  |  The optional argdefs tuple specifies the default argument values.
#  |  The optional closure tuple supplies the bindings for free variables.

def variadic(typ):
    """
    Make a function very-variadic.
    """
    def flatten(args):
        flat = []
        for arg in args:
            if isinstance(arg, typ):
                flat.append((arg,))
            else:
                flat.append(arg)
        return itertools.chain.from_iterable(flat)

    def decorator(wrapped):
        spec = inspect.getargspec(wrapped)
        name = wrapped.__name__

        assert spec.varargs is not None

        def call_wrapped(posargs, varargs, keywords={}):
            args = list(posargs)
            args.extend(flatten(varargs))
            return wrapped(*args, **keywords)

        # Example was generated with print ast.dump(ast.parse("def f(a, b, *args, **kwds): return call_wrapped((a, b), args, kwds)"), include_attributes=True)
        # http://code.activestate.com/recipes/578353-code-to-source-and-back/ helped a lot
        # http://stackoverflow.com/questions/10303248#29927459
        if sys.hexversion < 0x03000000:
            wrapper_ast_args = ast.arguments(
                    args=[ast.Name(id=a, ctx=ast.Param(), lineno=1, col_offset=0) for a in spec.args],
                    vararg=spec.varargs,
                    kwarg=spec.keywords,
                    defaults=[]
                )
        else:
            wrapper_ast_args = ast.arguments(
                args=[ast.arg(arg=a, annotation=None, lineno=1, col_offset=0) for a in spec.args],
                vararg=None if spec.varargs is None else ast.arg(arg=spec.varargs, annotation=None, lineno=1, col_offset=0),
                kwonlyargs=[],
                kw_defaults=[],
                kwarg=None if spec.keywords is None else ast.arg(arg=spec.keywords, annotation=None, lineno=1, col_offset=0),
                defaults=[]
            )
        wrapper_ast = ast.Module(body=[ast.FunctionDef(
            name=name,
            args=wrapper_ast_args,
            body=[ast.Return(value=ast.Call(
                func=ast.Name(id="call_wrapped", ctx=ast.Load(), lineno=1, col_offset=0),
                args=[
                    ast.Tuple(elts=[ast.Name(id=a, ctx=ast.Load(), lineno=1, col_offset=0) for a in spec.args], ctx=ast.Load(), lineno=1, col_offset=0),
                    ast.Name(id=spec.varargs, ctx=ast.Load(), lineno=1, col_offset=0),
                ] + ([] if spec.keywords is None else [ast.Name(id=spec.keywords, ctx=ast.Load(), lineno=1, col_offset=0)]),
                keywords=[], starargs=None, kwargs=None, lineno=1, col_offset=0
            ), lineno=1, col_offset=0)],
            decorator_list=[],
            lineno=1,
            col_offset=0
        )])
        wrapper_code = [c for c in compile(wrapper_ast, "<not_a_file>", "exec").co_consts if isinstance(c, types.CodeType)][0]
        wrapper = types.FunctionType(wrapper_code, {"call_wrapped": call_wrapped}, argdefs=spec.defaults)

        functools.update_wrapper(wrapper, wrapped)
        return wrapper
    return decorator

class PurelyVariadicFunctionTestCase(unittest.TestCase):
    def setUp(self):
        @variadic(int)
        def f(*xs):
            "f's doc"
            return xs
        self.f = f
        @variadic(int)
        def g(*ys):
            "g's doc"
            return ys
        self.g = g

    def test_name_is_preserved(self):
        self.assertEqual(self.f.__name__, "f")
        self.assertEqual(self.g.__name__, "g")

    def test_doc_is_preserved(self):
        self.assertEqual(self.f.__doc__, "f's doc")
        self.assertEqual(self.g.__doc__, "g's doc")

    def test_argspec_keeps_param_name(self):
        self.assertEqual(inspect.getargspec(self.f).varargs, "xs")
        self.assertEqual(inspect.getargspec(self.g).varargs, "ys")

    def test_call_without_arguments(self):
        self.assertEqual(self.f(), ())

    def test_call_with_one_argument(self):
        self.assertEqual(self.f(1), (1,))

    def test_call_with_several_arguments(self):
        self.assertEqual(self.f(1, 2, 3), (1, 2, 3))

    def test_call_with_one_list(self):
        self.assertEqual(self.f([1, 2, 3]), (1, 2, 3))

    def test_call_with_several_lists(self):
        self.assertEqual(self.f([1, 2], [3], [4, 5]), (1, 2, 3, 4, 5))

    def test_call_with_lists_and_arguments(self):
        self.assertEqual(self.f([1, 2], 3, 4, [5, 6], 7), (1, 2, 3, 4, 5, 6, 7))

    def test_call_with_keywords(self):
        with self.assertRaises(TypeError) as catcher:
            self.f(a=1)
        self.assertEqual(catcher.exception.args, ("f() got an unexpected keyword argument 'a'",))
        with self.assertRaises(TypeError) as catcher:
            self.g(a=1)
        self.assertEqual(catcher.exception.args, ("g() got an unexpected keyword argument 'a'",))

class NotOnlyVariadicFunctionTestCase(unittest.TestCase):
    def test_args_before_varargs(self):
        @variadic(int)
        def f(a, b, *xs):
            return a, b, xs
        self.assertEqual(f(1, 2, 3, [4, 5], 6), (1, 2, (3, 4, 5, 6)))

    @variadic(int)
    def f(self, a, b, *xs):
        return self, a, b, xs

    def test_method(self):
        self.assertEqual(self.f(1, 2, 3, [4, 5], 6), (self, 1, 2, (3, 4, 5, 6)))

    def test_kwds_after_varargs(self):
        @variadic(int)
        def f(a, b, *xs, **kwds):
            return a, b, xs, kwds
        self.assertEqual(f(1, 2, 3, [4, 5], 6, c=7, d=8), (1, 2, (3, 4, 5, 6), {"c": 7, "d": 8}))

    def test_defaults_on_args_before_varargs(self):
        default = object()  # To avoid implementations wich would stringify the default values and feed them to exec.
        @variadic(int)
        def f(a=None, b=default, *xs):
            return a, b, xs
        self.assertEqual(f(), (None, default, ()))

    def test_closures(self):
        a = 42
        @variadic(int)
        def f(*xs):
            return a, xs
        self.assertEqual(f(1, 2), (42, (1, 2)))
        a = 57
        self.assertEqual(f(1, 2), (57, (1, 2)))

if __name__ == "__main__":
    unittest.main()
<commit_msg>Remove intermediate wrapper<commit_after>#!/usr/bin/env python

# Copyright 2015 Vincent Jacques <vincent@vincent-jacques.net>

"""
Define a variadic function:

    >>> @variadic(int)
    ... def f(*xs):
    ...   return xs

It can be called with a variable number of arguments,
and they are passed to decorated function as a single, iterable, parameter:

    >>> f()
    ()
    >>> f(1)
    (1,)
    >>> f(1, 2, 3, 4)
    (1, 2, 3, 4)

But it can also be called with lists (any iterable, in fact) of arguments:

    >>> f([])
    ()
    >>> f([1, 2, 3], [4, 5, 6])
    (1, 2, 3, 4, 5, 6)
    >>> f(xrange(1, 4))
    (1, 2, 3)

And you can even mix them:

    >>> f(1, [2, 3], 4, xrange(5, 8))
    (1, 2, 3, 4, 5, 6, 7)
"""

import ast
import functools
import inspect
import itertools
import sys
import types
import unittest

# Todo-list:
# - allow passing the flatten function instead of typ
# - allow an even simpler usage without any parameters
# - add a parameter string to be prepended or appended to the docstring
# - support decorating callables that are not functions?
# - add a doctest showing stack trace when decorated function raises
# - autodoc a decorated function

# >>> help(types.FunctionType)
# class function(object)
#  |  function(code, globals[, name[, argdefs[, closure]]])
#  |
#  |  Create a function object from a code object and a dictionary.
#  |  The optional name string overrides the name from the code object.
#  |  The optional argdefs tuple specifies the default argument values.
#  |  The optional closure tuple supplies the bindings for free variables.

def variadic(typ):
    """
    Make a function very-variadic.
    """
    def flatten(args):
        flat = []
        for arg in args:
            if isinstance(arg, typ):
                flat.append((arg,))
            else:
                flat.append(arg)
        return itertools.chain.from_iterable(flat)

    def decorator(wrapped):
        spec = inspect.getargspec(wrapped)
        name = wrapped.__name__

        assert spec.varargs is not None

        # Example was generated with print ast.dump(ast.parse("def f(a, b, *args, **kwds): return call_wrapped((a, b), args, kwds)"), include_attributes=True)
        # http://code.activestate.com/recipes/578353-code-to-source-and-back/ helped a lot
        # http://stackoverflow.com/questions/10303248#29927459
        if sys.hexversion < 0x03000000:
            wrapper_ast_args = ast.arguments(
                    args=[ast.Name(id=a, ctx=ast.Param(), lineno=1, col_offset=0) for a in spec.args],
                    vararg=spec.varargs,
                    kwarg=spec.keywords,
                    defaults=[]
                )
        else:
            wrapper_ast_args = ast.arguments(
                args=[ast.arg(arg=a, annotation=None, lineno=1, col_offset=0) for a in spec.args],
                vararg=None if spec.varargs is None else ast.arg(arg=spec.varargs, annotation=None, lineno=1, col_offset=0),
                kwonlyargs=[],
                kw_defaults=[],
                kwarg=None if spec.keywords is None else ast.arg(arg=spec.keywords, annotation=None, lineno=1, col_offset=0),
                defaults=[]
            )
        wrapper_ast = ast.Module(body=[ast.FunctionDef(
            name=name,
            args=wrapper_ast_args,
            body=[ast.Return(value=ast.Call(
                func=ast.Name(id="wrapped", ctx=ast.Load(), lineno=1, col_offset=0),
                args=[ast.Name(id=a, ctx=ast.Load(), lineno=1, col_offset=0) for a in spec.args],
                keywords=[],
                starargs=ast.Call(
                    func=ast.Name(id="flatten", ctx=ast.Load(), lineno=1, col_offset=0),
                    args=[ast.Name(id=spec.varargs, ctx=ast.Load(), lineno=1, col_offset=0)],
                    keywords=[], starargs=None, kwargs=None, lineno=1, col_offset=0
                ),
                kwargs=None if spec.keywords is None else ast.Name(id=spec.keywords, ctx=ast.Load(), lineno=1, col_offset=0),
                lineno=1, col_offset=0
            ), lineno=1, col_offset=0)],
            decorator_list=[],
            lineno=1,
            col_offset=0
        )])
        wrapper_code = [c for c in compile(wrapper_ast, "<not_a_file>", "exec").co_consts if isinstance(c, types.CodeType)][0]
        wrapper = types.FunctionType(wrapper_code, {"wrapped": wrapped, "flatten": flatten}, argdefs=spec.defaults)

        functools.update_wrapper(wrapper, wrapped)
        return wrapper
    return decorator

class PurelyVariadicFunctionTestCase(unittest.TestCase):
    def setUp(self):
        @variadic(int)
        def f(*xs):
            "f's doc"
            return xs
        self.f = f
        @variadic(int)
        def g(*ys):
            "g's doc"
            return ys
        self.g = g

    def test_name_is_preserved(self):
        self.assertEqual(self.f.__name__, "f")
        self.assertEqual(self.g.__name__, "g")

    def test_doc_is_preserved(self):
        self.assertEqual(self.f.__doc__, "f's doc")
        self.assertEqual(self.g.__doc__, "g's doc")

    def test_argspec_keeps_param_name(self):
        self.assertEqual(inspect.getargspec(self.f).varargs, "xs")
        self.assertEqual(inspect.getargspec(self.g).varargs, "ys")

    def test_call_without_arguments(self):
        self.assertEqual(self.f(), ())

    def test_call_with_one_argument(self):
        self.assertEqual(self.f(1), (1,))

    def test_call_with_several_arguments(self):
        self.assertEqual(self.f(1, 2, 3), (1, 2, 3))

    def test_call_with_one_list(self):
        self.assertEqual(self.f([1, 2, 3]), (1, 2, 3))

    def test_call_with_several_lists(self):
        self.assertEqual(self.f([1, 2], [3], [4, 5]), (1, 2, 3, 4, 5))

    def test_call_with_lists_and_arguments(self):
        self.assertEqual(self.f([1, 2], 3, 4, [5, 6], 7), (1, 2, 3, 4, 5, 6, 7))

    def test_call_with_keywords(self):
        with self.assertRaises(TypeError) as catcher:
            self.f(a=1)
        self.assertEqual(catcher.exception.args, ("f() got an unexpected keyword argument 'a'",))
        with self.assertRaises(TypeError) as catcher:
            self.g(a=1)
        self.assertEqual(catcher.exception.args, ("g() got an unexpected keyword argument 'a'",))

class NotOnlyVariadicFunctionTestCase(unittest.TestCase):
    def test_args_before_varargs(self):
        @variadic(int)
        def f(a, b, *xs):
            return a, b, xs
        self.assertEqual(f(1, 2, 3, [4, 5], 6), (1, 2, (3, 4, 5, 6)))

    @variadic(int)
    def f(self, a, b, *xs):
        return self, a, b, xs

    def test_method(self):
        self.assertEqual(self.f(1, 2, 3, [4, 5], 6), (self, 1, 2, (3, 4, 5, 6)))

    def test_kwds_after_varargs(self):
        @variadic(int)
        def f(a, b, *xs, **kwds):
            return a, b, xs, kwds
        self.assertEqual(f(1, 2, 3, [4, 5], 6, c=7, d=8), (1, 2, (3, 4, 5, 6), {"c": 7, "d": 8}))

    def test_defaults_on_args_before_varargs(self):
        default = object()  # To avoid implementations wich would stringify the default values and feed them to exec.
        @variadic(int)
        def f(a=None, b=default, *xs):
            return a, b, xs
        self.assertEqual(f(), (None, default, ()))

    def test_closures(self):
        a = 42
        @variadic(int)
        def f(*xs):
            return a, xs
        self.assertEqual(f(1, 2), (42, (1, 2)))
        a = 57
        self.assertEqual(f(1, 2), (57, (1, 2)))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import json
from datetime import datetime

from flask import Response
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer
from sqlalchemy import asc

from groups import *

from ..models import CrashReport, UUID, Traceback, Application
from ..tools import save_report, delete_report as _delete_report
from ..extensions import search

@app.route('/reports/<int:report_number>')
@flask_login.login_required
def view_report(report_number):
    cr = CrashReport.query.filter(CrashReport.id == report_number).first()
    tracebacks = Traceback.query.filter(Traceback.crashreport_id == report_number).all()
    pylexer = PythonLexer(stripall=True)
    highlighted_source = []
    for tb in tracebacks:
        highlighted_line = [tb['Error Line Number'] - tb['Module Line Number'] + 1]
        htmlformatter = HtmlFormatter(linenos=True,
                                      cssclass='highlight',
                                      linenostart=tb['Module Line Number'],
                                      hl_lines=highlighted_line)
        highlighted_source.append(highlight(tb['Source Code'], pylexer, htmlformatter))

    alias = Alias.query.filter(Alias.uuid_id==cr.uuid_id).first()
    html = render_template('crashreport.html',
                           report=cr,
                           source_code=highlighted_source,
                           inspection_level=10000,
                           alias=alias,
                           search_links={'Google': search.get_search_link('google', cr.error_message),
                                         'StackOverflow': search.get_search_link('stackoverflow', cr.error_message)},
                           back_link=url_for('view_reports'))
    return html

@app.route('/reports/related/<int:report_id>')
@flask_login.login_required
def view_related_reports(report_id):
    return render_template('index.html', user=flask_login.current_user,
                    search_init="{related_to_id: %d}" % report_id)

@app.route('/reports/<int:report_number>/info', methods=['GET'])
@flask_login.login_required
def get_report_info(report_number):
    user = flask_login.current_user
    group = user.group
    aliases = {a.user_identifier: a.alias for a in group.aliases}
    report = CrashReport.query.filter(CrashReport.id == report_number).first()
    if report:
        return flask.jsonify(report_to_json(report, aliases=aliases))

def report_to_json(report, aliases=None):
    if aliases is None:
        user = report.uuid.user_identifier
    else:
        user = aliases.get(report.uuid.user_identifier, report.uuid.user_identifier)
    return {'report_number': report.id,
             'related_report_numbers': [r.id for r in report.related_reports],
             'application_name': report.application.name,
             'application_version': report.application.version_string,
             'is_release': report.application.is_release,
             'user': user,
             'error_type': report.error_type,
             'error_message': report.error_message,
             'date': report.date.strftime('%B %d %Y'),
             'time': report.date.strftime('%I:%M %p')
             }

@app.route('/reports/delete', methods=['POST'])
@flask_login.login_required
def delete_many_reports():
    data = request.get_json()
    report_numbers = data.get('report_numbers')
    delete_similar = request.args.get('delete_similar') == 'True' or data.get('delete_similar', False)
    success = _delete_report(delete_similar, *report_numbers)
    if success:
        response = 'Success. Crash report #%s deleted' % request.args.get('report_numbers')
    else:
        response = 'Failed. Crash report #%s does not exist.' % request.args.get('report_numbers')
    flash(response)
    return redirect(url_for('view_reports'))

@app.route('/reports/<int:report_id>/delete', methods=['POST'])
@flask_login.login_required
def delete_single_report(report_id, delete_similar=False):
    if request.data:
        data = request.get_json()
        delete_similar = data.get('delete_similar', delete_similar)
    delete_similar |= request.args.get('delete_similar') == 'True'
    success = _delete_report(delete_similar, report_id)
    if success:
        return 'Success. Crash report #%d deleted' % report_id
    else:
        return 'Failed. Crash report #%d does not exist.' % report_id

@app.route('/reports/upload', methods=['POST'])
def upload_single_report():
    if request.method == 'POST':
        payload = json.loads(request.data)
        cr, response = save_report(payload)
        return response
    else:
        return 'Upload failed'

@app.route('/reports/upload_many', methods=['POST'])
def upload_many_reports():
    if request.method == 'POST':
        payload = json.loads(request.data)
        for package in payload:
            cr, response = save_report(package)
        return response
    else:
        return 'Upload failed'

@app.route('/reports/view_stats', methods=['GET', 'POST'])
@flask_login.login_required
def view_report_stats():
    group = flask_login.current_user.group
    if group is None:
        return 'You are not in a group.'
    latest_applications = db.session.query(func.max(Application.version_0),
                                           func.max(Application.version_1),
                                           func.max(Application.version_2)) \
                                    .filter(Application.group_id==group.id,
                                            Application.is_release == True) \
                                    .group_by(Application.name).all()

    top_reports = {}
    for v0, v1, v2 in latest_applications:
        r = db.session.query(CrashReport, func.count(CrashReport.id).label('total'), func.count(CrashReport.uuid_id.distinct()))\
                      .group_by(CrashReport.related_group_id) \
                      .filter(CrashReport.group_id==group.id,
                              CrashReport.application.has(version_0=v0, version_1=v1, version_2=v2)) \
                      .order_by('total DESC').limit(5)
        if r:
            top_reports[r[0][0].application.name] = r

    html = render_template('report_statistics.html', user=flask_login.current_user, top_reports=top_reports)
    return html

@app.route('/reports/get_stats', methods=['GET'])
@flask_login.login_required
def get_report_stats():
    group = flask_login.current_user.group
    if request.args.get('type') == 'date':
        # Query for the number of reports for each day
        q = db.session.query(func.date(CrashReport.date), func.count(func.DATE(CrashReport.date)))\
                      .filter(CrashReport.group_id==group.id)
        # Only dates that are before today (so we take out units with time-stamps in the future)
        q = q.filter(CrashReport.date <= datetime.now())
        # Group / bin the results by day in ascending order
        if request.args.get('hide_aliased', 'false') == 'true':
            q = q.filter(CrashReport.uuid_id.notin_([a.uuid_id for a in flask_login.current_user.group.aliases]))
        if request.args.get('released_only', 'false') == 'true':
            q = q.join(Application).filter(Application.is_release == True)
        q = q.group_by(func.date(CrashReport.date)).order_by(asc(CrashReport.date))

        # Because SQLite does not support the Date class in queries, we cannot cast the results
        # Instead, do the casting ourselves
        data = []
        for datestr, n in q.all():
            d = datetime.strptime(datestr, '%Y-%m-%d')
            data.append((d.year, d.month-1, d.day, d.hour, n))

    elif request.args.get('type') == 'user':
        # https://stackoverflow.com/questions/25500904/counting-relationships-in-sqlalchemy/25525771#25525771
        q = db.session.query(UUID.user_identifier, func.count(CrashReport.id))\
                      .filter(CrashReport.group_id==group.id)\
                      .join(UUID.reports)\
                      .group_by(UUID.user_identifier)
        if request.args.get('hide_aliased', 'false') == 'true':
            q = q.filter(CrashReport.uuid_id.notin_([a.uuid_id for a in flask_login.current_user.group.aliases]))
        if request.args.get('released_only', 'false') == 'true':
            q = q.join(Application).filter(Application.is_release == True)
        data = q.all()
    else:
        return 'Invalid request'
    json_response = json.dumps(data)
    response = Response(json_response, content_type='application/json; charset=utf-8')
    response.headers.add('content-length', len(json_response))
    response.status_code = 200

    return response<commit_msg>view_report_stats() was not returning the latest application version but rather was finding the max of version 0-2 individually<commit_after>import json
from datetime import datetime

from flask import Response
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer
from sqlalchemy import asc

from groups import *

from ..models import CrashReport, UUID, Traceback, Application
from ..tools import save_report, delete_report as _delete_report
from ..extensions import search

@app.route('/reports/<int:report_number>')
@flask_login.login_required
def view_report(report_number):
    cr = CrashReport.query.filter(CrashReport.id == report_number).first()
    tracebacks = Traceback.query.filter(Traceback.crashreport_id == report_number).all()
    pylexer = PythonLexer(stripall=True)
    highlighted_source = []
    for tb in tracebacks:
        highlighted_line = [tb['Error Line Number'] - tb['Module Line Number'] + 1]
        htmlformatter = HtmlFormatter(linenos=True,
                                      cssclass='highlight',
                                      linenostart=tb['Module Line Number'],
                                      hl_lines=highlighted_line)
        highlighted_source.append(highlight(tb['Source Code'], pylexer, htmlformatter))

    alias = Alias.query.filter(Alias.uuid_id==cr.uuid_id).first()
    html = render_template('crashreport.html',
                           report=cr,
                           source_code=highlighted_source,
                           inspection_level=10000,
                           alias=alias,
                           search_links={'Google': search.get_search_link('google', cr.error_message),
                                         'StackOverflow': search.get_search_link('stackoverflow', cr.error_message)},
                           back_link=url_for('view_reports'))
    return html

@app.route('/reports/related/<int:report_id>')
@flask_login.login_required
def view_related_reports(report_id):
    return render_template('index.html', user=flask_login.current_user,
                    search_init="{related_to_id: %d}" % report_id)

@app.route('/reports/<int:report_number>/info', methods=['GET'])
@flask_login.login_required
def get_report_info(report_number):
    user = flask_login.current_user
    group = user.group
    aliases = {a.user_identifier: a.alias for a in group.aliases}
    report = CrashReport.query.filter(CrashReport.id == report_number).first()
    if report:
        return flask.jsonify(report_to_json(report, aliases=aliases))

def report_to_json(report, aliases=None):
    if aliases is None:
        user = report.uuid.user_identifier
    else:
        user = aliases.get(report.uuid.user_identifier, report.uuid.user_identifier)
    return {'report_number': report.id,
             'related_report_numbers': [r.id for r in report.related_reports],
             'application_name': report.application.name,
             'application_version': report.application.version_string,
             'is_release': report.application.is_release,
             'user': user,
             'error_type': report.error_type,
             'error_message': report.error_message,
             'date': report.date.strftime('%B %d %Y'),
             'time': report.date.strftime('%I:%M %p')
             }

@app.route('/reports/delete', methods=['POST'])
@flask_login.login_required
def delete_many_reports():
    data = request.get_json()
    report_numbers = data.get('report_numbers')
    delete_similar = request.args.get('delete_similar') == 'True' or data.get('delete_similar', False)
    success = _delete_report(delete_similar, *report_numbers)
    if success:
        response = 'Success. Crash report #%s deleted' % request.args.get('report_numbers')
    else:
        response = 'Failed. Crash report #%s does not exist.' % request.args.get('report_numbers')
    flash(response)
    return redirect(url_for('view_reports'))

@app.route('/reports/<int:report_id>/delete', methods=['POST'])
@flask_login.login_required
def delete_single_report(report_id, delete_similar=False):
    if request.data:
        data = request.get_json()
        delete_similar = data.get('delete_similar', delete_similar)
    delete_similar |= request.args.get('delete_similar') == 'True'
    success = _delete_report(delete_similar, report_id)
    if success:
        return 'Success. Crash report #%d deleted' % report_id
    else:
        return 'Failed. Crash report #%d does not exist.' % report_id

@app.route('/reports/upload', methods=['POST'])
def upload_single_report():
    if request.method == 'POST':
        payload = json.loads(request.data)
        cr, response = save_report(payload)
        return response
    else:
        return 'Upload failed'

@app.route('/reports/upload_many', methods=['POST'])
def upload_many_reports():
    if request.method == 'POST':
        payload = json.loads(request.data)
        for package in payload:
            cr, response = save_report(package)
        return response
    else:
        return 'Upload failed'

@app.route('/reports/view_stats', methods=['GET', 'POST'])
@flask_login.login_required
def view_report_stats():
    group = flask_login.current_user.group
    if group is None:
        return 'You are not in a group.'

    apps = db.session.query(Application.name)\
                     .filter(Application.group_id==group.id, Application.is_release == True)\
                     .group_by(Application.name).all()
    latest_applications = {}
    for (app,) in apps:
        (max_v0,) = db.session.query(func.max(Application.version_0)).filter(Application.name==app,
                                                                             Application.group_id == group.id,
                                                                             Application.is_release == True
                                                                             ).first()
        (max_v1,) = db.session.query(func.max(Application.version_1)).filter(Application.name==app,
                                                                             Application.group_id == group.id,
                                                                             Application.is_release == True,
                                                                             Application.version_0 == max_v0).first()
        _app, max_v2 = db.session.query(Application, func.max(Application.version_2))\
                                 .filter(Application.name==app,
                                         Application.group_id == group.id,
                                         Application.is_release == True,
                                         Application.version_0 == max_v0,
                                         Application.version_1 == max_v1).first()
        latest_applications[app] = max_v0, max_v1, max_v2

    top_reports = {}
    for app, (v0, v1, v2) in latest_applications.iteritems():
        r = db.session.query(CrashReport, func.count(CrashReport.id).label('total'), func.count(CrashReport.uuid_id.distinct()))\
                      .group_by(CrashReport.related_group_id) \
                      .filter(CrashReport.group_id==group.id,
                              CrashReport.application.has(version_0=v0, version_1=v1, version_2=v2)) \
                      .order_by('total DESC').limit(5).all()
        if r:
            top_reports[app] = r

    html = render_template('report_statistics.html', user=flask_login.current_user, top_reports=top_reports)
    return html

@app.route('/reports/get_stats', methods=['GET'])
@flask_login.login_required
def get_report_stats():
    group = flask_login.current_user.group
    if request.args.get('type') == 'date':
        # Query for the number of reports for each day
        q = db.session.query(func.date(CrashReport.date), func.count(func.DATE(CrashReport.date)))\
                      .filter(CrashReport.group_id==group.id)
        # Only dates that are before today (so we take out units with time-stamps in the future)
        q = q.filter(CrashReport.date <= datetime.now())
        # Group / bin the results by day in ascending order
        if request.args.get('hide_aliased', 'false') == 'true':
            q = q.filter(CrashReport.uuid_id.notin_([a.uuid_id for a in flask_login.current_user.group.aliases]))
        if request.args.get('released_only', 'false') == 'true':
            q = q.join(Application).filter(Application.is_release == True)
        q = q.group_by(func.date(CrashReport.date)).order_by(asc(CrashReport.date))

        # Because SQLite does not support the Date class in queries, we cannot cast the results
        # Instead, do the casting ourselves
        data = []
        for datestr, n in q.all():
            d = datetime.strptime(datestr, '%Y-%m-%d')
            data.append((d.year, d.month-1, d.day, d.hour, n))

    elif request.args.get('type') == 'user':
        # https://stackoverflow.com/questions/25500904/counting-relationships-in-sqlalchemy/25525771#25525771
        q = db.session.query(UUID.user_identifier, func.count(CrashReport.id))\
                      .filter(CrashReport.group_id==group.id)\
                      .join(UUID.reports)\
                      .group_by(UUID.user_identifier)
        if request.args.get('hide_aliased', 'false') == 'true':
            q = q.filter(CrashReport.uuid_id.notin_([a.uuid_id for a in flask_login.current_user.group.aliases]))
        if request.args.get('released_only', 'false') == 'true':
            q = q.join(Application).filter(Application.is_release == True)
        data = q.all()
    else:
        return 'Invalid request'
    json_response = json.dumps(data)
    response = Response(json_response, content_type='application/json; charset=utf-8')
    response.headers.add('content-length', len(json_response))
    response.status_code = 200

    return response<|endoftext|>