<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import json
from mock import patch

from eduid_common.api.testing import EduidAPITestCase
from eduid_common.api.utils import retrieve_modified_ts
from eduid_webapp.email.app import email_init_app

class EmailTests(EduidAPITestCase):

    def load_app(self, config):
        """
        Called from the parent class, so we can provide the appropiate flask
        app for this test case.
        """
        return email_init_app('testing', config)

    def update_config(self, config):
        config.update({
            'AVAILABLE_LANGUAGES': {'en': 'English','sv': 'Svenska'},
            'MSG_BROKER_URL': 'amqp://dummy',
            'AM_BROKER_URL': 'amqp://dummy',
            'CELERY_CONFIG': {
                'CELERY_RESULT_BACKEND': 'amqp',
                'CELERY_TASK_SERIALIZER': 'json'
            },
        })
        return config

    def init_data(self):
        self.app.dashboard_userdb.save(self.test_user, check_sync=False)
        retrieve_modified_ts(self.test_user)

    def test_get_all_emails(self):
        response = self.browser.get('/all')
        self.assertEqual(response.status_code, 302)  # Redirect to token service

        eppn = self.test_user_data['eduPersonPrincipalName']
        with self.session_cookie(self.browser, eppn) as client:
            response2 = client.get('/all')

            self.assertEqual(response.status_code, 302)

            email_data = json.loads(response2.data)

            self.assertEqual(email_data['type'], 'GET_EMAIL_ALL_SUCCESS')
            self.assertEqual(email_data['payload']['emails'][0].get('email'), 'johnsmith@example.com')
            self.assertEqual(email_data['payload']['emails'][0].get('confirmed'), False)
            self.assertEqual(email_data['payload']['emails'][1].get('email'), 'johnsmith2@example.com')
            self.assertEqual(email_data['payload']['emails'][1].get('confirmed'), False)

    def test_post_email_error_no_data(self):
        response = self.browser.post('/new')
        self.assertEqual(response.status_code, 302) # Redirect to token service

        eppn = self.test_user_data['eduPersonPrincipalName']
        with self.session_cookie(self.browser, eppn) as client:
            response2 = client.post('/new')

            self.assertEqual(response.status_code, 302)

            new_email_data = json.loads(response2.data)
            self.assertEqual(new_email_data['type'], 'POST_EMAIL_NEW_FAIL')

    @patch('eduid_common.api.am.AmRelay.request_user_sync')
    def test_post_email(self, mock_request_user_sync):
        response = self.browser.post('/new')
        self.assertEqual(response.status_code, 302)  # Redirect to token service

        mock_request_user_sync.return_value = True
        eppn = self.test_user_data['eduPersonPrincipalName']

        with self.session_cookie(self.browser, eppn) as client:
            data = {
                'email': 'john-smith@example.com',
                'confirmed': False,
                'primary': False,
            }
            response2 = client.post('/new', data=json.dumps(data),
                                   content_type=self.content_type_json)

            self.assertEqual(response2.status_code, 200)

            new_email_data = json.loads(response2.data)

            self.assertEqual(new_email_data['type'], 'POST_EMAIL_NEW_SUCCESS')
            self.assertEqual(new_email_data['payload']['emails'][2].get('email'), 'john-smith@example.com')
            self.assertEqual(new_email_data['payload']['emails'][2].get('confirmed'), False)
<commit_msg>Added email tests<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import json
from mock import patch

from eduid_common.api.testing import EduidAPITestCase
from eduid_common.api.utils import retrieve_modified_ts
from eduid_webapp.email.app import email_init_app

class EmailTests(EduidAPITestCase):

    def load_app(self, config):
        """
        Called from the parent class, so we can provide the appropiate flask
        app for this test case.
        """
        return email_init_app('testing', config)

    def update_config(self, config):
        config.update({
            'AVAILABLE_LANGUAGES': {'en': 'English','sv': 'Svenska'},
            'MSG_BROKER_URL': 'amqp://dummy',
            'AM_BROKER_URL': 'amqp://dummy',
            'CELERY_CONFIG': {
                'CELERY_RESULT_BACKEND': 'amqp',
                'CELERY_TASK_SERIALIZER': 'json'
            },
        })
        return config

    def init_data(self):
        self.app.dashboard_userdb.save(self.test_user, check_sync=False)
        retrieve_modified_ts(self.test_user)

    def test_get_all_emails(self):
        response = self.browser.get('/all')
        self.assertEqual(response.status_code, 302)  # Redirect to token service

        eppn = self.test_user_data['eduPersonPrincipalName']
        with self.session_cookie(self.browser, eppn) as client:
            response2 = client.get('/all')

            self.assertEqual(response.status_code, 302)

            email_data = json.loads(response2.data)

            self.assertEqual(email_data['type'], 'GET_EMAIL_ALL_SUCCESS')
            self.assertEqual(email_data['payload']['emails'][0].get('email'), 'johnsmith@example.com')
            self.assertEqual(email_data['payload']['emails'][0].get('confirmed'), False)
            self.assertEqual(email_data['payload']['emails'][1].get('email'), 'johnsmith2@example.com')
            self.assertEqual(email_data['payload']['emails'][1].get('confirmed'), False)

    def test_post_email_error_no_data(self):
        response = self.browser.post('/new')
        self.assertEqual(response.status_code, 302) # Redirect to token service

        eppn = self.test_user_data['eduPersonPrincipalName']
        with self.session_cookie(self.browser, eppn) as client:
            response2 = client.post('/new')

            self.assertEqual(response.status_code, 302)

            new_email_data = json.loads(response2.data)
            self.assertEqual(new_email_data['type'], 'POST_EMAIL_NEW_FAIL')

    @patch('eduid_common.api.am.AmRelay.request_user_sync')
    def test_post_email(self, mock_request_user_sync):
        response = self.browser.post('/new')
        self.assertEqual(response.status_code, 302)  # Redirect to token service

        mock_request_user_sync.return_value = True
        eppn = self.test_user_data['eduPersonPrincipalName']

        with self.session_cookie(self.browser, eppn) as client:
            data = {
                'email': 'john-smith@example.com',
                'confirmed': False,
                'primary': False,
            }
            response2 = client.post('/new', data=json.dumps(data),
                                   content_type=self.content_type_json)

            self.assertEqual(response2.status_code, 200)

            new_email_data = json.loads(response2.data)

            self.assertEqual(new_email_data['type'], 'POST_EMAIL_NEW_SUCCESS')
            self.assertEqual(new_email_data['payload']['emails'][2].get('email'), 'john-smith@example.com')
            self.assertEqual(new_email_data['payload']['emails'][2].get('confirmed'), False)

    @patch('eduid_common.api.am.AmRelay.request_user_sync')
    def test_post_primary(self, mock_request_user_sync):
        response = self.browser.post('/primary')
        self.assertEqual(response.status_code, 302)  # Redirect to token service

        mock_request_user_sync.return_value = True
        eppn = self.test_user_data['eduPersonPrincipalName']

        with self.session_cookie(self.browser, eppn) as client:
            data = {
                'email': 'johnsmith@example.com',
                'confirmed': True,
                'primary': True,
            }

            response2 = client.post('/primary', data=json.dumps(data),
                                    content_type=self.content_type_json)

            self.assertEqual(response2.status_code, 200)

            new_email_data = json.loads(response2.data)

            self.assertEqual(new_email_data['type'], 'POST_EMAIL_PRIMARY_SUCCESS')
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse_lazy
from django.shortcuts import render, get_object_or_404
from django.views.generic import (
    ListView, DetailView,
    CreateView, UpdateView, DeleteView
)

from . import models

def team_list(request):
    teams = models.Team.objects.all()
    return render(request, 'teams/team_list.html', {'teams': teams})

def team_detail(request, pk):
    team = get_object_or_404(models.Team, pk=pk)
    return render(request, 'teams/team_detail.html', {'team': team})

class TeamListView(CreateView, ListView):
    context_object_name = "teams"
    fields = ("name", "practice_location", "coach")
    model = models.Team
    template_name = 'teams/team_list.html'

class TeamDetailView(DetailView, UpdateView):
    fields = ("name", "practice_location", "coach")
    model = models.Team
    template_name = "teams/team_detail.html"

class TeamCreateView(CreateView):
    model = models.Team
    fields = ('name', 'practice_location', 'coach')

    def get_initial(self):
        initial = super().get_initial()
        initial["coach"] = self.request.user.pk
        return initial

class TeamUpdateView(UpdateView):
    model = models.Team
    fields = ('name', 'practice_location', 'coach')

class TeamDeleteView(DeleteView):
    model = models.Team
    success_url = reverse_lazy("teams:list")

    def get_queryset(self):
        if not self.request.user.is_superuser:
            return self.model.objects.filter(coach=self.request.user)
        return self.model.object.all()<commit_msg>Django class-based views: add login required mixin<commit_after>from django.contrib.auth.mixins import LoginRequiredMixin
from django.core.urlresolvers import reverse_lazy
from django.shortcuts import render, get_object_or_404
from django.views.generic import (
    ListView, DetailView,
    CreateView, UpdateView, DeleteView
)

from . import models

def team_list(request):
    teams = models.Team.objects.all()
    return render(request, 'teams/team_list.html', {'teams': teams})

def team_detail(request, pk):
    team = get_object_or_404(models.Team, pk=pk)
    return render(request, 'teams/team_detail.html', {'team': team})

class TeamListView(CreateView, ListView):
    context_object_name = "teams"
    fields = ("name", "practice_location", "coach")
    model = models.Team
    template_name = 'teams/team_list.html'

class TeamDetailView(DetailView, UpdateView):
    fields = ("name", "practice_location", "coach")
    model = models.Team
    template_name = "teams/team_detail.html"

class TeamCreateView(LoginRequiredMixin, CreateView):
    model = models.Team
    fields = ('name', 'practice_location', 'coach')

    def get_initial(self):
        initial = super().get_initial()
        initial["coach"] = self.request.user.pk
        return initial

class TeamUpdateView(LoginRequiredMixin, UpdateView):
    model = models.Team
    fields = ('name', 'practice_location', 'coach')

class TeamDeleteView(LoginRequiredMixin, DeleteView):
    model = models.Team
    success_url = reverse_lazy("teams:list")

    def get_queryset(self):
        if not self.request.user.is_superuser:
            return self.model.objects.filter(coach=self.request.user)
        return self.model.object.all()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.core.paginator import Paginator as DjangoPaginator, Page as DjangoPage
from copy import copy
import math

class PaginatorMixin(object):
    '''
    PaginatorMixin, Mixin to DjangoPaginator, subclass need attr page_num.
    '''

    def __init__(self, display_pages=10, curr_page=1, **kwargs):
        self.display_pages = display_pages
        self.curr_page = curr_page

class RequestPaginatorMixin(PaginatorMixin):
    def __init__(self, first_text = u'first', last_text = u'last',
                show_first = True, show_last = True,
                show_prev = True, show_next = True,
                prev_page_text = u'prev page', next_page_text = u'next page',
                auto_hide_prev = True, auto_hide_next = True,
                *args, **kwargs):
        self.first_text = first_text
        self.last_text = last_text
        self.show_first = show_first
        self.show_last = show_last
        self.show_prev = show_prev
        self.show_next = show_next
        self.prev_page_text = prev_page_text
        self.next_page_text = next_page_text
        self.auto_hide_prev = auto_hide_prev
        self.auto_hide_next = auto_hide_next
        super(RequestPaginatorMixin, self).__init__(*args, **kwargs)

class Paginator(DjangoPaginator, PaginatorMixin):
    def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True, **kwargs):
        PaginatorMixin.__init__(self, **kwargs)
        DjangoPaginator.__init__(self, object_list, per_page, orphans, allow_empty_first_page)

    @property
    def page_range(self):
        '''
        if you display_pages = 10, curr_page = 10
        page_range will be [5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
        '''
        start = max(1, self.curr_page - int(math.ceil(self.display_pages / 2)))
        end = start + self.display_pages - 1
        if end > self.num_pages:
            end = self.num_pages
            start = max(1, self.num_pages - self.display_pages)
        return range(start, end+1)
<commit_msg>add RequestPaginator<commit_after># -*- coding: utf-8 -*-
import math
from copy import copy
from urllib import urlencode

from django.core.paginator import Page as DjangoPage
from django.core.paginator import Paginator as DjangoPaginator
from django.core.paginator import EmptyPage

class PaginatorMixin(object):
    '''
    PaginatorMixin, Mixin to DjangoPaginator, subclass need attr page_num.
    '''

    def __init__(self, display_pages=10, curr_page=1, **kwargs):
        self.display_pages = display_pages
        self.curr_page = curr_page

class RequestPaginatorMixin(PaginatorMixin):
    def __init__(self, request=None, page_num_param=u'page',
                first_text=u'first', last_text=u'last',
                show_first=True, show_last=True,
                show_prev=True, show_next=True,
                prev_page_text=u'prev page', next_page_text=u'next page',
                auto_hide_prev=True, auto_hide_next=True,
                url_path='',
                *args, **kwargs):
        assert request, 'request must not be None'
        self._request = request
        if not url_path:
            url_path = request.path_info
        self.url_path = url_path
        self.get_params = request.GET.dict()
        self.page_num_param = page_num_param
        self.first_text = first_text
        self.last_text = last_text
        self.show_first = show_first
        self.show_last = show_last
        self.show_prev = show_prev
        self.show_next = show_next
        self.prev_page_text = prev_page_text
        self.next_page_text = next_page_text
        self.auto_hide_prev = auto_hide_prev
        self.auto_hide_next = auto_hide_next
        if 'curr_page' not in kwargs:
            curr_page = self.get_params.get(page_num_param)
            if curr_page:
                kwargs['curr_page'] = curr_page

        super(RequestPaginatorMixin, self).__init__(*args, **kwargs)

def validate_number_patch(paginator, curr_page):
    if curr_page < 1:
        if paginator.allow_empty_first_page:
            p.curr_page = 1
    if curr_page > paginator.num_pages:
       paginator.curr_page = paginator.num_pages

def paginator_page_range(paginator):
    '''
    if you display_pages = 10, curr_page = 10
    page_range will be [5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
    '''
    start = max(1, paginator.curr_page - int(math.ceil(paginator.display_pages / 2)))
    end = start + paginator.display_pages - 1
    if end > paginator.num_pages:
        end = paginator.num_pages
        start = max(1, paginator.num_pages - paginator.display_pages + 1)
    return range(start, end+1)

class Paginator(DjangoPaginator, PaginatorMixin):

    def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True, **kwargs):
        PaginatorMixin.__init__(self, **kwargs)
        DjangoPaginator.__init__(self, object_list, per_page, orphans, allow_empty_first_page)
        validate_number_patch(self, self.curr_page)

    @property
    def page_range(self):
        return paginator_page_range(self)

    @property
    def page(self):
        validate_number_patch(self, self.curr_page)
        return super(Paginator, self).page(self.curr_page)

class RequestPaginator(DjangoPaginator, RequestPaginatorMixin):

    def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True, **kwargs):
        RequestPaginatorMixin.__init__(self, **kwargs)
        DjangoPaginator.__init__(self, object_list, per_page, orphans, allow_empty_first_page)
        validate_number_patch(self, self.curr_page)

    @property
    def page_range(self):
        '''
        if you display_pages = 10, curr_page = 10
        page_range will be [5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
        '''
        _range = paginator_page_range(self)
        _page_range = []
        _params = copy(self.get_params)
        for i in _range:
            _params[self.page_num_param] = i
            _page_range.append('{}?{}'.format(self.url_path, urlencode(_params)))
        return _page_range

    @property
    def page(self):
        return super(RequestPaginator, self).page(self.curr_page)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Automatic building of SeqAn apps and releases.
"""

import subprocess
import optparse
import os.path
import sys
import shutil
import tempfile

# The SVN command to use.
SVN_BINARY='svn'
# The CMake command to use.
CMAKE_BINARY='cmake'

# The default value for the SVN tags.
DEFAULT_TAGS_URL='http://svn.seqan.de/seqan/tags'
# The default minimal revision of that tags must have.
DEFAULT_START_REVISION=13708
# The path to the package repository.
DEFAULT_PACKAGE_DB='.'

class MinisculeSvnWrapper(object):
    """Minimal SVN wrapper."""

    def ls(self, url):
        """Execute 'svn ls ${url}'."""
        print >>sys.stderr, 'Executing "%s %s %s"' % (SVN_BINARY, 'ls -v', url)
        popen = subprocess.Popen([SVN_BINARY, 'ls', '-v', url],
                                 stdout=subprocess.PIPE)
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during SVN call.'
            return 1
        lines = out_data.splitlines()
        revs_tags = [(int(line.split()[0]), line.split()[-1]) for line in lines]
        res = []
        for rev, tag in revs_tags:
            if tag == './':
                continue  # Skip dot-slash.
            tag2 = tag[:-1]
            res.append((rev, tag2))
        return res

    def co(self, url, dest_dir):
        """Execute 'svn co ${url} ${dest_dir}'."""
        print >>sys.stderr, 'Executing "%s %s %s %s"' % (SVN_BINARY, 'co', url, dest_dir)
        popen = subprocess.Popen([SVN_BINARY, 'co', url, dest_dir])
        popen.wait()
        return popen.returncode

class Package(object):
    """Represent a package with a given name, version, OS and architeture."""

    def __init__(self, name, version, os, word_size, pkg_format):
        self.name = name
        self.version = version
        self.os = os
        self.word_size = word_size
        SYS_NAMES = {'Windows': {'32': 'win32-i686', '64': 'win64-x86'},
                     'Linux': {'32': 'Linux-i686', '64': 'Linux-x86_64'},
                     'Mac': {'32': 'Mac-i686', '64': 'Mac-x86_64'}}
        self.system_name = SYS_NAMES[os][word_size]
        self.pkg_format = pkg_format

    def fileName(self):
        return '%s-%s-%s.%s' % (self.name, self.version, self.system_name,
                                self.pkg_format)

class BuildStep(object):
    """Management of one build step."""

    def __init__(self, path, name, version, os, word_size, pkg_formats,
                 svn_url, make_args, options, tmp_dir=None):
        self.base_path = path
        self.name = name
        self.version = version  # TODO(holtgrew): Unused, overwritten below.
        self.major_version = int(version.split('.')[0])
        self.minor_version = int(version.split('.')[1])
        self.patch_version = 0
        if len(version.split('.')) > 2:
            self.patch_version = int(version.split('.')[2])
        self.version = '%d.%d.%d' % (self.major_version, self.minor_version, self.patch_version)
        self.os = os
        self.word_size = word_size
        self.pkg_formats = pkg_formats
        if name == 'seqan':
            self.packages = [Package(name + suffix, self.version, os, word_size, f)
                             for f in pkg_formats for suffix in ['-apps', '-library']]
        else:
            self.packages = [Package(name, self.version, os, word_size, f)
                             for f in pkg_formats]
        self.svn_url = svn_url
        self.make_args = make_args
        self.options = options
        # If set then this is used instead of a random name in TMPDIR.
        self.tmp_dir = tmp_dir

    def buildNeeded(self):
        """Returns whether one of the package files is missing."""
        for p in self.packages:
            package_path = os.path.join(self.base_path, p.fileName())
            if not os.path.exists(package_path):
                return True
            elif self.options.verbosity >= 1:
                print >>sys.stderr, 'File %s exists.' % package_path
        return False

    def copyArchives(self, build_dir):
        """Copy build packages to base_path directory."""
        for p in self.packages:
            from_ = os.path.join(build_dir, p.fileName())
            if os.path.exists(from_):
                to = os.path.join(self.base_path, os.path.basename(from_))
                print >>sys.stderr, "Copying %s => %s" % (from_, to)
                if 'x86' in to and 'x86_64' not in to:  # fix processor name
                    to = to.replace('x86', 'x86_64')
                if 'win32' in to or 'win64' in to:  # fix OS name
                    to = to.replace('win32', 'Windows').replace('win64', 'Windows')
                shutil.copyfile(from_, to)
            else:
                print >>sys.stderr, "%s does not exist (not fatal)" % from_

    def buildSeqAnRelease(self, checkout_dir, build_dir):
        """Build SeqAn release: Apps and library build."""
        # Build seqan-apps.
        #
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY,
                      "-DSEQAN_BUILD_PACKAGE=SEQAN_RELEASE_APPS"]
        if self.os in ['Linux', 'Mac']:
            cmake_args += ['-m%s' % self.word_size]
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        # Execute Make.
        make_args = [MAKE_BINARY, "package"] + self.make_args
        print >>sys.stderr, 'Executing Make: "%s"' % (' '.join(make_args),)
        # Copy over the archives.
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)
        # Build seqan-library.
        #
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY,
                      "-DSEQAN_BUILD_SYSTEM=SEQAN_RELEASE_LIBRARY"]
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        # Execute Make.
        make_args = [MAKE_BINARY, "package"] + self.make_args
        print >>sys.stderr, 'Executing Make: "%s"' % (' '.join(make_args),)
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)

    def buildApp(self, checkout_dir, build_dir):
        """Build an application."""
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        if not os.path.exists(build_dir):
            os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY, checkout_dir,# '-G', 'Visual Studio 10',
                      "-DCMAKE_BUILD_TYPE=Release",
                      "-DSEQAN_BUILD_SYSTEM=APP:%s" % self.name,
                      "-DSEQAN_APP_VERSION=%d.%d.%d" %
                        (self.major_version, self.minor_version, self.patch_version)]
        # Use appropriate CMake flags for OS and processor.
        if self.word_size == '32':
            cmake_args.append('-DSEQAN_SYSTEM_PROCESSOR=i686')
            if self.os != 'Windows':
                cmake_args.append('-DCMAKE_CXX_FLAGS=-m32')
            else:
                cmake_args += ['-G', 'Visual Studio 10']
        else:  # self.word_size == '64'
            if self.os == 'Windows':
                cmake_args += ['-G', 'Visual Studio 10 Win64']
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        #for key in sorted(os.environ.keys()):
        #    print key, ': ', os.environ[key]
        popen = subprocess.Popen(cmake_args, cwd=build_dir, env=os.environ.copy())
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during make call.'
            print out_data
            print err_data
            return 1
        # Build and package project.
        make_args = [CMAKE_BINARY, '--build', build_dir, '--target', 'package']
        if self.options.verbosity > 1:
            make_args.insert(1, 'VERBOSE=1')
        print >>sys.stderr, 'Building with CMake: "%s"' % (' '.join(make_args),)
        popen = subprocess.Popen(make_args, cwd=build_dir)
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during make call.'
            print out_data
            print err_data
            return 1
        # Copy out archives.
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)

    def tmpDir(self):
      print 'self.tmp_dir = %s' % self.tmp_dir
      if self.tmp_dir:
        if not os.path.exists(self.tmp_dir):
          os.makedirs(self.tmp_dir)
        return self.tmp_dir
      else:
        return tempfile.mkdtemp()

    def execute(self):
        """Execute build step."""
        # Create temporary directory.
        tmp_dir = self.tmpDir()
        print >>sys.stderr, 'Temporary directory is %s' % (tmp_dir,)
        # Create SVN checkout in temporary directory.
        checkout_dir = os.path.join(tmp_dir, os.path.basename(self.svn_url))
        print >>sys.stderr, 'Creating checkout in %s' % checkout_dir
        svn = MinisculeSvnWrapper()
        svn.co(self.svn_url, checkout_dir)
        # Create build directory.
        suffix = '-build-%s-%s' % (self.os, self.word_size)
        build_dir = os.path.join(tmp_dir, os.path.basename(self.svn_url) + suffix)
        if os.path.exists(build_dir):
          print >>sys.stderr, 'Removing build directory %s' % (build_dir,)
          shutil.rmtree(build_dir)
        # Perform the build.  We have to separate between app and whole SeqAn releases.
        if self.name == 'seqan':
            self.buildSeqAnRelease(checkout_dir, build_dir)
        else:
            self.buildApp(checkout_dir, build_dir)
        # Remove temporary directory again.
        if not self.tmp_dir:  # Only remove it not set with --tmp-dir.
          print >>sys.stderr, 'Removing temporary directory %s' % (tmp_dir,)
          shutil.rmtree(tmp_dir)

def work(options):
    """Run the individual steps."""
    # Get the revisions and tag names.
    svn = MinisculeSvnWrapper()
    revs_tags = [(rev, tag) for (rev, tag) in svn.ls(options.tags_url)
                 if rev >= options.start_revision and
                    '-' in tag]
    # Enumerate all package names that we could enumerate.
    print 'revs_tags = %s' % revs_tags
    print 'word_sizes = %s' % options.word_sizes
    for rev, tag in revs_tags:
        name, version = tag.rsplit('-', 1)
        for word_size in options.word_sizes.split(','):
            # Create build step for this package name.
            pkg_formats = options.package_formats.split(',')
            svn_url = options.tags_url + '/' + tag
            build_step = BuildStep(options.package_db, name, version, options.os,
                                   word_size, pkg_formats, svn_url,
                                   options.make_args.split(), options, options.tmp_dir)
            # Check whether we need to build this.
            if not build_step.buildNeeded():
                continue  # Skip
            # Execute build step.
            build_step.execute()
    return 0

def main():
    """Program entry point."""
    # Parse Arguments.
    parser = optparse.OptionParser()
    parser.add_option('-t', '--tags-url', dest='tags_url',
                      default=DEFAULT_TAGS_URL,
                      help='This URL is searched for tags.', metavar='URL')
    parser.add_option('--package-db', dest='package_db', type='string',
                      default=DEFAULT_PACKAGE_DB,
                      help='Path the directory with the packages.')
    parser.add_option('-s', '--start-revision', dest='start_revision',
                      default=DEFAULT_START_REVISION,
                      type='int', help='Ignore all tags with smaller revision.')
    parser.add_option('-v', dest='verbosity', action='count', default=1,
                      help='Increase verbosity.')
    parser.add_option('--package-formats', dest='package_formats',
                      default='tar.bz2,zip',
                      help='Expect the following packages to be created.')
    parser.add_option('--os', dest='os', help='Expect the packages to be created for this OS.',
                      default='Linux')
    parser.add_option('--word-sizes', dest='word_sizes', default='32,64',
                      help='Build binaries with the given word sizes')
    parser.add_option('--make-args', dest='make_args', type="string", default='',
                      help='Arguments for make.')
    parser.add_option('--tmp-dir', dest='tmp_dir', type='string', default=None,
                      help='Temporary directory to use. Use this to reuse the same checkout.')
    parser.epilog = ('The program will use the environment variable TMPDIR as '
                     'the directory for temporary files.')

    options, args = parser.parse_args()
    if args:
        parser.error('No arguments expected!')
        return 1

    # Fire up work.
    print >>sys.stderr, 'Running SeqAn Auto Builder'
    return work(options)
<commit_msg>[FIX] Renaming of package names also when checking whether a build is needed.<commit_after>#!/usr/bin/env python
"""
Automatic building of SeqAn apps and releases.
"""

import subprocess
import optparse
import os.path
import sys
import shutil
import tempfile

# The SVN command to use.
SVN_BINARY='svn'
# The CMake command to use.
CMAKE_BINARY='cmake'

# The default value for the SVN tags.
DEFAULT_TAGS_URL='http://svn.seqan.de/seqan/tags'
# The default minimal revision of that tags must have.
DEFAULT_START_REVISION=13708
# The path to the package repository.
DEFAULT_PACKAGE_DB='.'

class MinisculeSvnWrapper(object):
    """Minimal SVN wrapper."""

    def ls(self, url):
        """Execute 'svn ls ${url}'."""
        print >>sys.stderr, 'Executing "%s %s %s"' % (SVN_BINARY, 'ls -v', url)
        popen = subprocess.Popen([SVN_BINARY, 'ls', '-v', url],
                                 stdout=subprocess.PIPE)
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during SVN call.'
            return 1
        lines = out_data.splitlines()
        revs_tags = [(int(line.split()[0]), line.split()[-1]) for line in lines]
        res = []
        for rev, tag in revs_tags:
            if tag == './':
                continue  # Skip dot-slash.
            tag2 = tag[:-1]
            res.append((rev, tag2))
        return res

    def co(self, url, dest_dir):
        """Execute 'svn co ${url} ${dest_dir}'."""
        print >>sys.stderr, 'Executing "%s %s %s %s"' % (SVN_BINARY, 'co', url, dest_dir)
        popen = subprocess.Popen([SVN_BINARY, 'co', url, dest_dir])
        popen.wait()
        return popen.returncode

class Package(object):
    """Represent a package with a given name, version, OS and architeture."""

    def __init__(self, name, version, os, word_size, pkg_format):
        self.name = name
        self.version = version
        self.os = os
        self.word_size = word_size
        SYS_NAMES = {'Windows': {'32': 'win32-i686', '64': 'win64-x86'},
                     'Linux': {'32': 'Linux-i686', '64': 'Linux-x86_64'},
                     'Mac': {'32': 'Mac-i686', '64': 'Mac-x86_64'}}
        self.system_name = SYS_NAMES[os][word_size]
        self.pkg_format = pkg_format

    def fileName(self):
        return '%s-%s-%s.%s' % (self.name, self.version, self.system_name,
                                self.pkg_format)

class BuildStep(object):
    """Management of one build step."""

    def __init__(self, path, name, version, os, word_size, pkg_formats,
                 svn_url, make_args, options, tmp_dir=None):
        self.base_path = path
        self.name = name
        self.version = version  # TODO(holtgrew): Unused, overwritten below.
        self.major_version = int(version.split('.')[0])
        self.minor_version = int(version.split('.')[1])
        self.patch_version = 0
        if len(version.split('.')) > 2:
            self.patch_version = int(version.split('.')[2])
        self.version = '%d.%d.%d' % (self.major_version, self.minor_version, self.patch_version)
        self.os = os
        self.word_size = word_size
        self.pkg_formats = pkg_formats
        if name == 'seqan':
            self.packages = [Package(name + suffix, self.version, os, word_size, f)
                             for f in pkg_formats for suffix in ['-apps', '-library']]
        else:
            self.packages = [Package(name, self.version, os, word_size, f)
                             for f in pkg_formats]
        self.svn_url = svn_url
        self.make_args = make_args
        self.options = options
        # If set then this is used instead of a random name in TMPDIR.
        self.tmp_dir = tmp_dir

    def buildNeeded(self):
        """Returns whether one of the package files is missing."""
        for p in self.packages:
            package_path = os.path.join(self.base_path, p.fileName())
            if 'x86' in package_path and 'x86_64' not in package_path:  # fix processor name
                package_path = package_path.replace('x86', 'x86_64')
            if 'win32' in package_path or 'win64' in package_path:  # fix OS name
                package_path = package_path.replace('win32', 'Windows').replace('win64', 'Windows')
            if not os.path.exists(package_path):
                return True
            elif self.options.verbosity >= 1:
                print >>sys.stderr, 'File %s exists.' % package_path
        return False

    def copyArchives(self, build_dir):
        """Copy build packages to base_path directory."""
        for p in self.packages:
            from_ = os.path.join(build_dir, p.fileName())
            if os.path.exists(from_):
                to = os.path.join(self.base_path, os.path.basename(from_))
                print >>sys.stderr, "Copying %s => %s" % (from_, to)
                if 'x86' in to and 'x86_64' not in to:  # fix processor name
                    to = to.replace('x86', 'x86_64')
                if 'win32' in to or 'win64' in to:  # fix OS name
                    to = to.replace('win32', 'Windows').replace('win64', 'Windows')
                shutil.copyfile(from_, to)
            else:
                print >>sys.stderr, "%s does not exist (not fatal)" % from_

    def buildSeqAnRelease(self, checkout_dir, build_dir):
        """Build SeqAn release: Apps and library build."""
        # Build seqan-apps.
        #
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY,
                      "-DSEQAN_BUILD_PACKAGE=SEQAN_RELEASE_APPS"]
        if self.os in ['Linux', 'Mac']:
            cmake_args += ['-m%s' % self.word_size]
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        # Execute Make.
        make_args = [MAKE_BINARY, "package"] + self.make_args
        print >>sys.stderr, 'Executing Make: "%s"' % (' '.join(make_args),)
        # Copy over the archives.
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)
        # Build seqan-library.
        #
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY,
                      "-DSEQAN_BUILD_SYSTEM=SEQAN_RELEASE_LIBRARY"]
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        # Execute Make.
        make_args = [MAKE_BINARY, "package"] + self.make_args
        print >>sys.stderr, 'Executing Make: "%s"' % (' '.join(make_args),)
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)

    def buildApp(self, checkout_dir, build_dir):
        """Build an application."""
        # Create build directory.
        print >>sys.stderr, "Creating build directory %s" % (build_dir,)
        if not os.path.exists(build_dir):
            os.mkdir(build_dir)
        # Execute CMake.
        cmake_args = [CMAKE_BINARY, checkout_dir,# '-G', 'Visual Studio 10',
                      "-DCMAKE_BUILD_TYPE=Release",
                      "-DSEQAN_BUILD_SYSTEM=APP:%s" % self.name,
                      "-DSEQAN_APP_VERSION=%d.%d.%d" %
                        (self.major_version, self.minor_version, self.patch_version)]
        # Use appropriate CMake flags for OS and processor.
        if self.word_size == '32':
            cmake_args.append('-DSEQAN_SYSTEM_PROCESSOR=i686')
            if self.os != 'Windows':
                cmake_args.append('-DCMAKE_CXX_FLAGS=-m32')
            else:
                cmake_args += ['-G', 'Visual Studio 10']
        else:  # self.word_size == '64'
            if self.os == 'Windows':
                cmake_args += ['-G', 'Visual Studio 10 Win64']
        print >>sys.stderr, 'Executing CMake: "%s"' % (' '.join(cmake_args),)
        #for key in sorted(os.environ.keys()):
        #    print key, ': ', os.environ[key]
        popen = subprocess.Popen(cmake_args, cwd=build_dir, env=os.environ.copy())
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during make call.'
            print out_data
            print err_data
            return 1
        # Build and package project.
        make_args = [CMAKE_BINARY, '--build', build_dir, '--target', 'package']
        if self.options.verbosity > 1:
            make_args.insert(1, 'VERBOSE=1')
        print >>sys.stderr, 'Building with CMake: "%s"' % (' '.join(make_args),)
        popen = subprocess.Popen(make_args, cwd=build_dir)
        out_data, err_data = popen.communicate()
        if popen.returncode != 0:
            print >>sys.stderr, 'ERROR during make call.'
            print out_data
            print err_data
            return 1
        # Copy out archives.
        self.copyArchives(build_dir)
        # Remove build directory.
        print >>sys.stderr, 'Removing build directory %s' % build_dir
        shutil.rmtree(build_dir)

    def tmpDir(self):
      print 'self.tmp_dir = %s' % self.tmp_dir
      if self.tmp_dir:
        if not os.path.exists(self.tmp_dir):
          os.makedirs(self.tmp_dir)
        return self.tmp_dir
      else:
        return tempfile.mkdtemp()

    def execute(self):
        """Execute build step."""
        # Create temporary directory.
        tmp_dir = self.tmpDir()
        print >>sys.stderr, 'Temporary directory is %s' % (tmp_dir,)
        # Create SVN checkout in temporary directory.
        checkout_dir = os.path.join(tmp_dir, os.path.basename(self.svn_url))
        print >>sys.stderr, 'Creating checkout in %s' % checkout_dir
        svn = MinisculeSvnWrapper()
        svn.co(self.svn_url, checkout_dir)
        # Create build directory.
        suffix = '-build-%s-%s' % (self.os, self.word_size)
        build_dir = os.path.join(tmp_dir, os.path.basename(self.svn_url) + suffix)
        if os.path.exists(build_dir):
          print >>sys.stderr, 'Removing build directory %s' % (build_dir,)
          shutil.rmtree(build_dir)
        # Perform the build.  We have to separate between app and whole SeqAn releases.
        if self.name == 'seqan':
            self.buildSeqAnRelease(checkout_dir, build_dir)
        else:
            self.buildApp(checkout_dir, build_dir)
        # Remove temporary directory again.
        if not self.tmp_dir:  # Only remove it not set with --tmp-dir.
          print >>sys.stderr, 'Removing temporary directory %s' % (tmp_dir,)
          shutil.rmtree(tmp_dir)

def work(options):
    """Run the individual steps."""
    # Get the revisions and tag names.
    svn = MinisculeSvnWrapper()
    revs_tags = [(rev, tag) for (rev, tag) in svn.ls(options.tags_url)
                 if rev >= options.start_revision and
                    '-' in tag]
    # Enumerate all package names that we could enumerate.
    print 'revs_tags = %s' % revs_tags
    print 'word_sizes = %s' % options.word_sizes
    for rev, tag in revs_tags:
        name, version = tag.rsplit('-', 1)
        for word_size in options.word_sizes.split(','):
            # Create build step for this package name.
            pkg_formats = options.package_formats.split(',')
            svn_url = options.tags_url + '/' + tag
            build_step = BuildStep(options.package_db, name, version, options.os,
                                   word_size, pkg_formats, svn_url,
                                   options.make_args.split(), options, options.tmp_dir)
            # Check whether we need to build this.
            if not build_step.buildNeeded():
                continue  # Skip
            # Execute build step.
            build_step.execute()
    return 0

def main():
    """Program entry point."""
    # Parse Arguments.
    parser = optparse.OptionParser()
    parser.add_option('-t', '--tags-url', dest='tags_url',
                      default=DEFAULT_TAGS_URL,
                      help='This URL is searched for tags.', metavar='URL')
    parser.add_option('--package-db', dest='package_db', type='string',
                      default=DEFAULT_PACKAGE_DB,
                      help='Path the directory with the packages.')
    parser.add_option('-s', '--start-revision', dest='start_revision',
                      default=DEFAULT_START_REVISION,
                      type='int', help='Ignore all tags with smaller revision.')
    parser.add_option('-v', dest='verbosity', action='count', default=1,
                      help='Increase verbosity.')
    parser.add_option('--package-formats', dest='package_formats',
                      default='tar.bz2,zip',
                      help='Expect the following packages to be created.')
    parser.add_option('--os', dest='os', help='Expect the packages to be created for this OS.',
                      default='Linux')
    parser.add_option('--word-sizes', dest='word_sizes', default='32,64',
                      help='Build binaries with the given word sizes')
    parser.add_option('--make-args', dest='make_args', type="string", default='',
                      help='Arguments for make.')
    parser.add_option('--tmp-dir', dest='tmp_dir', type='string', default=None,
                      help='Temporary directory to use. Use this to reuse the same checkout.')
    parser.epilog = ('The program will use the environment variable TMPDIR as '
                     'the directory for temporary files.')

    options, args = parser.parse_args()
    if args:
        parser.error('No arguments expected!')
        return 1

    # Fire up work.
    print >>sys.stderr, 'Running SeqAn Auto Builder'
    return work(options)
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 rPath, Inc.
#
# All Rights Reserved
#

import re
from xml.dom import minidom

from django import http

from xobj import xobj

from mint.django_rest.rbuilder import modellib
from mint.django_rest.rbuilder.inventory import models

def str_to_underscore(name):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

def to_underscore(node):
    for name in ['tagName', 'nodeName']:
        if hasattr(node, name):
            setattr(node, name, str_to_underscore(getattr(node, name)))

    for child in node.childNodes:
        for name in ['tagName', 'nodeName']:
            if hasattr(child, name):
                setattr(child, name, str_to_underscore(getattr(child, name)))
                to_underscore(child)

def str_to_camel_case(name):
    def repl(m):
        return m.group()[:-2] + m.group()[-1].upper()
    s1 = re.sub('(.)_([a-z])', repl, name)
    return s1

def to_camel_case(node):
    for name in ['tagName', 'nodeName']:
        if hasattr(node, name):
            setattr(node, name, str_to_camel_case(getattr(node, name)))

    for child in node.childNodes:
        for name in ['tagName', 'nodeName']:
            if hasattr(child, name):
                setattr(child, name, str_to_camel_case(getattr(child, name)))
                to_camel_case(child)

def requires(model_name, save=True):
    """
    Decorator that parses the post data on a request into the class
    specified by modelClass.
    """
    def decorate(function):

        def inner(*args, **kw):
            request = args[1]
            xml = request.raw_post_data
            doc = minidom.parseString(xml)
            root_node = doc.documentElement
            to_underscore(root_node)
            underscore_xml = doc.toxml(encoding='UTF-8')
            built_model = xobj.parse(underscore_xml)#, documentClass=DjangoDocument, typeMap=models.type_map)
            model_xml = str_to_underscore(model_name)
            built_model = getattr(built_model, model_xml)
            model = modellib.type_map[model_name].objects.load_from_object(
                built_model, request, save=save)
            # loaded_model = models.type_map[model_name].objects.load(built_model)
            # if not loaded_model:
                # built_model.save()
                # loaded_model = built_model
            # loaded_model.set_related()
            kw[model_name] = model
            return function(*args, **kw)

        return inner
    return decorate

def return_xml(function):
    """
    Decorator that serializes a returned parser object into xml with a root
    node of modelName.
    """
    def inner(*args, **kw):
        ret_val = function(*args, **kw)
        if isinstance(ret_val, http.HttpResponse):
            return ret_val
        response = http.HttpResponse()
        response['Content-Type'] = 'text/xml'
        request = args[1]
        xml = ret_val.to_xml(request)
        doc = minidom.parseString(xml)
        to_camel_case(doc.documentElement)
        response.write(doc.toxml(encoding='UTF-8'))
        return response

    return inner
<commit_msg>remove crufty commented out code<commit_after>#
# Copyright (c) 2010 rPath, Inc.
#
# All Rights Reserved
#

import re
from xml.dom import minidom

from django import http

from xobj import xobj

from mint.django_rest.rbuilder import modellib
from mint.django_rest.rbuilder.inventory import models

def str_to_underscore(name):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

def to_underscore(node):
    for name in ['tagName', 'nodeName']:
        if hasattr(node, name):
            setattr(node, name, str_to_underscore(getattr(node, name)))

    for child in node.childNodes:
        for name in ['tagName', 'nodeName']:
            if hasattr(child, name):
                setattr(child, name, str_to_underscore(getattr(child, name)))
                to_underscore(child)

def str_to_camel_case(name):
    def repl(m):
        return m.group()[:-2] + m.group()[-1].upper()
    s1 = re.sub('(.)_([a-z])', repl, name)
    return s1

def to_camel_case(node):
    for name in ['tagName', 'nodeName']:
        if hasattr(node, name):
            setattr(node, name, str_to_camel_case(getattr(node, name)))

    for child in node.childNodes:
        for name in ['tagName', 'nodeName']:
            if hasattr(child, name):
                setattr(child, name, str_to_camel_case(getattr(child, name)))
                to_camel_case(child)

def requires(model_name, save=True):
    """
    Decorator that parses the post data on a request into the class
    specified by modelClass.
    """
    def decorate(function):

        def inner(*args, **kw):
            request = args[1]
            xml = request.raw_post_data
            doc = minidom.parseString(xml)
            root_node = doc.documentElement
            to_underscore(root_node)
            underscore_xml = doc.toxml(encoding='UTF-8')
            built_model = xobj.parse(underscore_xml)
            model_xml = str_to_underscore(model_name)
            built_model = getattr(built_model, model_xml)
            model = modellib.type_map[model_name].objects.load_from_object(
                built_model, request, save=save)
            kw[model_name] = model
            return function(*args, **kw)

        return inner
    return decorate

def return_xml(function):
    """
    Decorator that serializes a returned parser object into xml with a root
    node of modelName.
    """
    def inner(*args, **kw):
        ret_val = function(*args, **kw)
        if isinstance(ret_val, http.HttpResponse):
            return ret_val
        response = http.HttpResponse()
        response['Content-Type'] = 'text/xml'
        request = args[1]
        xml = ret_val.to_xml(request)
        doc = minidom.parseString(xml)
        to_camel_case(doc.documentElement)
        response.write(doc.toxml(encoding='UTF-8'))
        return response

    return inner
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2006-2016  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia import *
from essentia.utils import *
from essentia.standard import *
from essentia import array as esarr
import numpy as np
import os

EssentiaException = RuntimeError

def find_files(directory, pattern):
    for root, dirs, files in os.walk(directory):
        for basename in files:
            if basename.lower().endswith(pattern):
                filename = os.path.join(root, basename)
                yield filename

test_types = [
    'values',  # n-dimensional values. (e,g. MFCC)
    'events',  # detecting events (e.g, onsets)
    'hybrid',  # events with a value (e.g, peaks with amplitude)
    'bool',  # True/False test (e.g, presence of voice)
    'semantic',  # high level descriptor (e.g, key)
]

default_audio_types = ('wav', 'mp3', 'flac', 'ogg')  # To be extended

class QaTest:
    wrappers = dict()
    data = dict()
    routes = dict()
    metrics = dict()

    solutions = dict()
    ground_true = dict()
    times = dict()
    scores = dict()

    fs = 44100.  # global fs for the test
    test_type = ''  # todo: restrict the possible values
    time_wrappers = True
    verbose = True
    log = True

    def __init__(self, test_type='values', wrappers=[], metrics=[], time_wrappers=True, verbose=False, log=True):
        """

        :param test_type: {value, values, events, hybrid, bool}
        :param wrappers: list of QaWrapper objects to provide solutions
        :param metrics: list of QaMetric objects to assess the solutions
        :param time_wrappers: If true the Wrappers are timed.
        """

        self.verbose = verbose
        self.set_test_type(test_type)
        self.set_wrappers(wrappers)
        self.set_metrics(metrics)
        self.time_wrappers = time_wrappers
        self.log = log

    def set_test_type(self, test_type):
        self.assert_test_type(test_type)
        self.test_type = test_type
        if self.verbose:
            print 'test_type is ok'

    def set_wrappers(self, wrappers):
        self.assert_wrappers(wrappers)
        for wrapper in wrappers:
            self.wrappers[wrapper.__class__.__name__] = wrapper
        if self.verbose:
            print 'Wrappers seem ok'

    def load_audio(self, filename, pattern=default_audio_types, stereo=False):
        """
        Uses Essentia MonoLoader to load the audio data. If filename is a folder it will look for all the `.extension` \
        files in the folder.
        :param filename:
        :param pattern:
        :return:
        """
        if not os.path.isfile(filename):
            files = [x for x in find_files(filename, pattern)]
        else:
            files = [filename]
        if len(files) == 0:
            raise EssentiaException("The file does not exist or the folder doesn't contain any '{}' file".format(pattern))
        try:
            for f in files:
                name = ''.join(os.path.basename(f).split('.')[:-1])
                if stereo:
                    audio, fs, _, _, _, _ = AudioLoader(filename=f)()
                    if np.abs(fs - self.fs) > 1e-3:
                        audio = Resample(inputSampleRate=fs, outputSampleRate=self.fs)(audio)
                    self.data[name] = audio
                else:
                    self.data[name] = MonoLoader(filename=f, sampleRate=self.fs)()
                self.routes[name] = f
        except IOError:
            if self.verbose:
                print 'cannot open {}'.format(f)  # todo improve exception handling
        # else:
        #    raise EssentiaException

    def set_metrics(self, metrics):
        if not type(metrics) == list:
            metrics = [metrics]
        self.assert_metrics(metrics)
        for metric in metrics:
            self.metrics[metric.__class__.__name__] = metric
        if self.verbose:
            print 'metrics seem ok'

    def add_metrics(self, metrics):
        if not type(metrics) == list:
            metrics = [metrics]
        self.assert_metrics(metrics)
        self.metrics.add(metrics)
        if self.verbose:
            print 'metrics seem ok'

    def load_solution(self, filename, name='', ground_true=False):
        if not os.path.isfile(filename):
            files = [x for x in find_files(filename, '')]
        else:
            files = [filename]

        for f in files:
            try:
                instance = ''.join(os.path.basename(f).split('.')[:-1])
                ext = (f.split('.')[-1])

                # Exclude known audio types so audio everything can be in the same folder
                if ext in default_audio_types:
                    continue

                parsers = {
                    'svl': lambda x: self.load_svl(x),
                    'csv': lambda x: self.load_csv(x),
                    'lab': lambda x: self.load_lab(x)
                    }

                try:
                    points = parsers[ext](f)
                except KeyError:
                    if self.verbose:
                        print ("ERROR: {} not loaded. \n"
                               "'{}' extension is not supported yet. Try with one of the supperted formats {}"
                               .format(f, ext, parsers.keys()))
                    continue

                if ground_true:
                    self.ground_true[instance] = np.array(points)
                else:
                    if name == '':
                        raise EssentiaException('If this solution is not the ground true it should be given a name')
                    self.solutions[name, instance] = points
            except EssentiaException:
                continue

    def load_svl(self):
        print 'This menthod is not implemeted in this class. Override it'
        return  # How should a general implementation be?

    def load_lab(self):
        print 'This menthod is not implemeted in this class. Override it'
        return  # How should a general implementation be?

    def compute(self, key_wrap, wrapper, key_inst, instance):
        if self.verbose:
            print "Computing file '{}' with the wrapper '{}'...".format(key_inst, key_wrap)
        self.solutions[key_wrap, key_inst] = wrapper.compute(self, instance, key_inst, key_wrap)

    def compute_and_time(self, key_wrap, wrapper, key_inst, instance):
        if self.verbose:
            print "Computing file '{}' with the wrapper '{}'...".format(key_inst, key_wrap)
        solution, time = wrapper.compute_and_time(self, instance, key_inst, key_wrap)
        self.solutions[key_wrap, key_inst] = solution
        self.times[key_wrap, key_inst] = time

    def compute_all(self, output_file='compute.log'):
        for key_wrap, wrapper in self.wrappers.iteritems():
            for key_inst, instance in self.data.iteritems():
                if self.time_wrappers:
                    self.compute_and_time(key_wrap, wrapper, key_inst, instance)
                else:
                    self.compute(key_wrap, wrapper, key_inst, instance)

        if self.log:
            np.set_printoptions(precision=3)
            final_text = []
            for data_key in self.data.keys():
                text = ['Results for: {}\n'.format(data_key)]
                found_something = False
                for wrapper_key in self.wrappers.keys():
                    if self.solutions[(wrapper_key, data_key)].any():
                        found_something = True
                        text.append('Using {}:'.format(wrapper_key))
                        text.append(str(self.solutions[(wrapper_key, data_key)]))
                        text.append('\n')
                if found_something:
                    final_text.append(''.join(text))
            with open(output_file, 'w') as o_file:
                o_file.write(''.join(final_text))
            if self.time_wrappers:
                self.compare_elapsed_times(output_file=output_file, mode='a')

    def compare_elapsed_times(self, output_file='stats.log', mode='a'):
        w_names = self.wrappers.keys()
        i_names = self.data.keys()

        arrs = np.array([np.array([self.times[w, i] for i in i_names]) for w in w_names])
        means = np.mean(arrs, 1)

        fastest_idx = np.argmin(means)
        text = []
        text.append('')
        text.append('*' * 70)
        text.append('{} is the fastest method. Lasted {:.3f}s on average'
                    .format(w_names[fastest_idx], means[fastest_idx]))

        for i in range(len(w_names)):
            if i != fastest_idx:
                text.append('{} lasted {:.3f}s. {:.2f}x slower'
                            .format(w_names[i], means[i], means[i] / means[fastest_idx]))
        text.append('*' * 70)
        text.append('')

        if self.verbose:
            print '\n'.join(text)

        with open(output_file, mode) as o_file:
            o_file.write('\n'.join(text))

    def score(self, key_wrap, key_inst, solution, key_metric, metric, gt):
        if self.verbose:
            print "Scoring file '{}' computed with the wrapper '{}' using metric '{}'..." \
                .format(key_inst, key_wrap, key_metric)
        self.scores[key_wrap, key_inst, key_metric] = metric.score(gt, solution)

    def score_all(self):
        gt_flags = []
        gt_name = ''
        for key_wrap, wrapper in self.wrappers.iteritems():
            gt_flags.append(wrapper.ground_true)
            if wrapper.ground_true:
                gt_name = wrapper.name

        if not bool(self.ground_true):

            #  assert only one GT
            if sum(gt_flags) > 1:
                raise EssentiaException("Only one wrapper can be set as ground true")
            if sum(gt_flags) == 0:
                raise EssentiaException("No wrapper is set as ground true")
        else:
            if sum(gt_flags) != 0:
                if self.verbose:
                    print("warning: When ground truth is set externally the ground truth wrapper would be ignored")

        for key_sol, solution in self.solutions.iteritems():
            key_wrap, key_inst = key_sol
            gt = self.ground_true.get(key_inst, None)
            if gt is not None:
                for key_metric, metric in self.metrics.iteritems():
                    self.score(key_wrap, key_inst, solution, key_metric, metric, gt)
                continue

            gt = self.solutions.get((gt_name, key_inst), None)
            if gt is not None:
                if key_wrap != gt_name:
                    for key_metric, metric in self.metrics.iteritems():
                        self.score(key_wrap, key_inst, solution, key_metric, metric, gt)
                    continue
            if self.verbose:
                print "{} was not scored because there is not ground truth available".format(key_inst)

    def save_test(self, output_file):
        import pickle
        with open('{}.pkl'.format(output_file), 'wb') as f:
            pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)

    def remove_wrapper(self, name):
        if name in self.wrappers:
            del self.wrappers[name]
        else:
            print 'No wrapper named {} found'.format(name)

    def clear_wrappers(self):
        self.wrappers.clear()

    def remove_solution(self, name):
        if name in self.solutions:
            del self.solutions[name]
        else:
            print 'No solution named {} found'.format(name)

    def clear_solutions(self, wrapper=''):
        if wrapper is '':
            self.solutions.clear()
        else:
            if wrapper in self.wrappers:
                names = self.filter(self.solutions.keys(), 0, wrapper)
                for name in names:
                    self.remove_solution(name)
            else:
                print 'No wrapper named {} found'.format(wrapper)

    def remove_scores(self, name):
        if name in self.scores:
            del self.solutions[name]
        else:
            print 'No score named {} found'.format(name)

    def clear_scores(self, wrapper='', metric=''):
        if wrapper is '' and metric is '':
            self.solutions.clear()
        else:
            names = self.solutions.keys()
            if wrapper in self.wrappers:
                names = self.filter(names, 0, wrapper)

            if metric in self.metrics:
                names = self.filter(names, 2, metric)

            for name in names:
                self.remove_solution(name)

    @staticmethod
    def load_test(input_file):
        import pickle
        with open('{}.pkl'.format(input_file), 'rb') as f:
            return pickle.load(f)

    @staticmethod
    def assert_wrappers(wrappers):
        types = set()
        for wrapper in wrappers:
            if not isinstance(wrapper, QaWrapper):
                raise EssentiaException('Input should be wrappers')
            types.add(wrapper.test_type)
        if len(types) > 1:
            raise EssentiaException("Wrappers type don't match")

    @staticmethod
    def assert_test_type(test_type):
        if test_type.lower() not in test_types:
            raise EssentiaException("'{}' is not supported among the test types ({})"
                                    .format(test_type, test_types))

    @staticmethod
    def assert_metrics(metrics):
        for metric in metrics:
            if not isinstance(metric, QaMetric):
                raise EssentiaException('Metrics should be wrapped in a QaMetric object')

    @staticmethod
    def filter(tuples, pos, patt):
        aux = [idx for idx, x in enumerate(tuples) if x[pos] == patt]
        filtered = []

        for idx in aux:
            filtered.append(tuples[idx])
        return filtered

class QaWrapper:
    test_type = ''
    ground_true = False
    name = ''

    def __init__(self, test_type='values', ground_true=False):
        QaTest.assert_test_type(test_type)
        self.test_type = test_type
        self.ground_true = ground_true
        self.name = self.__class__.__name__
        pass

    def compute(self):
        """
        This method should be overloaded in the sibling class.
        :return:
        """
        pass

    def compute_and_time(self, *args):
        from time import clock

        start = clock()
        solution = self.compute(*args)
        end = clock()

        secs = (end - start)

        return solution, secs

class QaMetric:
    def __init__(self):
        pass

    def score(self):
        """
        This method should be overloaded in the sibling class.
        :return:
        """
        pass
<commit_msg>changed timing module to `timeit` (subprocess support)<commit_after>#!/usr/bin/env python

# Copyright (C) 2006-2016  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia import *
from essentia.utils import *
from essentia.standard import *
from essentia import array as esarr
import numpy as np
import os

EssentiaException = RuntimeError

def find_files(directory, pattern):
    for root, dirs, files in os.walk(directory):
        for basename in files:
            if basename.lower().endswith(pattern):
                filename = os.path.join(root, basename)
                yield filename

test_types = [
    'values',  # n-dimensional values. (e,g. MFCC)
    'events',  # detecting events (e.g, onsets)
    'hybrid',  # events with a value (e.g, peaks with amplitude)
    'bool',  # True/False test (e.g, presence of voice)
    'semantic',  # high level descriptor (e.g, key)
]

default_audio_types = ('wav', 'mp3', 'flac', 'ogg')  # To be extended

class QaTest:
    wrappers = dict()
    data = dict()
    routes = dict()
    metrics = dict()

    solutions = dict()
    ground_true = dict()
    times = dict()
    scores = dict()

    fs = 44100.  # global fs for the test
    test_type = ''  # todo: restrict the possible values
    time_wrappers = True
    verbose = True
    log = True

    def __init__(self, test_type='values', wrappers=[], metrics=[], time_wrappers=True, verbose=False, log=True):
        """

        :param test_type: {value, values, events, hybrid, bool}
        :param wrappers: list of QaWrapper objects to provide solutions
        :param metrics: list of QaMetric objects to assess the solutions
        :param time_wrappers: If true the Wrappers are timed.
        """

        self.verbose = verbose
        self.set_test_type(test_type)
        self.set_wrappers(wrappers)
        self.set_metrics(metrics)
        self.time_wrappers = time_wrappers
        self.log = log

    def set_test_type(self, test_type):
        self.assert_test_type(test_type)
        self.test_type = test_type
        if self.verbose:
            print 'test_type is ok'

    def set_wrappers(self, wrappers):
        self.assert_wrappers(wrappers)
        for wrapper in wrappers:
            self.wrappers[wrapper.__class__.__name__] = wrapper
        if self.verbose:
            print 'Wrappers seem ok'

    def load_audio(self, filename, pattern=default_audio_types, stereo=False):
        """
        Uses Essentia MonoLoader to load the audio data. If filename is a folder it will look for all the `.extension` \
        files in the folder.
        :param filename:
        :param pattern:
        :return:
        """
        if not os.path.isfile(filename):
            files = [x for x in find_files(filename, pattern)]
        else:
            files = [filename]
        if len(files) == 0:
            raise EssentiaException("The file does not exist or the folder doesn't contain any '{}' file".format(pattern))
        try:
            for f in files:
                name = ''.join(os.path.basename(f).split('.')[:-1])
                if stereo:
                    audio, fs, _, _, _, _ = AudioLoader(filename=f)()
                    if np.abs(fs - self.fs) > 1e-3:
                        audio = Resample(inputSampleRate=fs, outputSampleRate=self.fs)(audio)
                    self.data[name] = audio
                else:
                    self.data[name] = MonoLoader(filename=f, sampleRate=self.fs)()
                self.routes[name] = f
        except IOError:
            if self.verbose:
                print 'cannot open {}'.format(f)  # todo improve exception handling
        # else:
        #    raise EssentiaException

    def set_metrics(self, metrics):
        if not type(metrics) == list:
            metrics = [metrics]
        self.assert_metrics(metrics)
        for metric in metrics:
            self.metrics[metric.__class__.__name__] = metric
        if self.verbose:
            print 'metrics seem ok'

    def add_metrics(self, metrics):
        if not type(metrics) == list:
            metrics = [metrics]
        self.assert_metrics(metrics)
        self.metrics.add(metrics)
        if self.verbose:
            print 'metrics seem ok'

    def load_solution(self, filename, name='', ground_true=False):
        if not os.path.isfile(filename):
            files = [x for x in find_files(filename, '')]
        else:
            files = [filename]

        for f in files:
            try:
                instance = ''.join(os.path.basename(f).split('.')[:-1])
                ext = (f.split('.')[-1])

                # Exclude known audio types so audio everything can be in the same folder
                if ext in default_audio_types:
                    continue

                parsers = {
                    'svl': lambda x: self.load_svl(x),
                    'csv': lambda x: self.load_csv(x),
                    'lab': lambda x: self.load_lab(x)
                    }

                try:
                    points = parsers[ext](f)
                except KeyError:
                    if self.verbose:
                        print ("ERROR: {} not loaded. \n"
                               "'{}' extension is not supported yet. Try with one of the supperted formats {}"
                               .format(f, ext, parsers.keys()))
                    continue

                if ground_true:
                    self.ground_true[instance] = np.array(points)
                else:
                    if name == '':
                        raise EssentiaException('If this solution is not the ground true it should be given a name')
                    self.solutions[name, instance] = points
            except EssentiaException:
                continue

    def load_svl(self):
        print 'This menthod is not implemeted in this class. Override it'
        return  # How should a general implementation be?

    def load_lab(self):
        print 'This menthod is not implemeted in this class. Override it'
        return  # How should a general implementation be?

    def compute(self, key_wrap, wrapper, key_inst, instance):
        if self.verbose:
            print "Computing file '{}' with the wrapper '{}'...".format(key_inst, key_wrap)
        self.solutions[key_wrap, key_inst] = wrapper.compute(self, instance, key_inst, key_wrap)

    def compute_and_time(self, key_wrap, wrapper, key_inst, instance):
        if self.verbose:
            print "Computing file '{}' with the wrapper '{}'...".format(key_inst, key_wrap)
        solution, time = wrapper.compute_and_time(self, instance, key_inst, key_wrap)
        self.solutions[key_wrap, key_inst] = solution
        self.times[key_wrap, key_inst] = time

    def compute_all(self, output_file='compute.log'):
        for key_wrap, wrapper in self.wrappers.iteritems():
            for key_inst, instance in self.data.iteritems():
                if self.time_wrappers:
                    self.compute_and_time(key_wrap, wrapper, key_inst, instance)
                else:
                    self.compute(key_wrap, wrapper, key_inst, instance)

        if self.log:
            np.set_printoptions(precision=3)
            final_text = []
            for data_key in self.data.keys():
                text = ['Results for: {}\n'.format(data_key)]
                found_something = False
                for wrapper_key in self.wrappers.keys():
                    if self.solutions[(wrapper_key, data_key)].any():
                        found_something = True
                        text.append('Using {}:'.format(wrapper_key))
                        text.append(str(self.solutions[(wrapper_key, data_key)]))
                        text.append('\n')
                if found_something:
                    final_text.append(''.join(text))
            with open(output_file, 'w') as o_file:
                o_file.write(''.join(final_text))
            if self.time_wrappers:
                self.compare_elapsed_times(output_file=output_file, mode='a')

    def compare_elapsed_times(self, output_file='stats.log', mode='a'):
        w_names = self.wrappers.keys()
        i_names = self.data.keys()

        arrs = np.array([np.array([self.times[w, i] for i in i_names]) for w in w_names])
        means = np.mean(arrs, 1)

        fastest_idx = np.argmin(means)
        text = []
        text.append('')
        text.append('*' * 70)
        text.append('{} is the fastest method. Lasted {:.3f}s on average'
                    .format(w_names[fastest_idx], means[fastest_idx]))

        for i in range(len(w_names)):
            if i != fastest_idx:
                text.append('{} lasted {:.3f}s. {:.2f}x slower'
                            .format(w_names[i], means[i], means[i] / means[fastest_idx]))
        text.append('*' * 70)
        text.append('')

        if self.verbose:
            print '\n'.join(text)

        with open(output_file, mode) as o_file:
            o_file.write('\n'.join(text))

    def score(self, key_wrap, key_inst, solution, key_metric, metric, gt):
        if self.verbose:
            print "Scoring file '{}' computed with the wrapper '{}' using metric '{}'..." \
                .format(key_inst, key_wrap, key_metric)
        self.scores[key_wrap, key_inst, key_metric] = metric.score(gt, solution)

    def score_all(self):
        gt_flags = []
        gt_name = ''
        for key_wrap, wrapper in self.wrappers.iteritems():
            gt_flags.append(wrapper.ground_true)
            if wrapper.ground_true:
                gt_name = wrapper.name

        if not bool(self.ground_true):

            #  assert only one GT
            if sum(gt_flags) > 1:
                raise EssentiaException("Only one wrapper can be set as ground true")
            if sum(gt_flags) == 0:
                raise EssentiaException("No wrapper is set as ground true")
        else:
            if sum(gt_flags) != 0:
                if self.verbose:
                    print("warning: When ground truth is set externally the ground truth wrapper would be ignored")

        for key_sol, solution in self.solutions.iteritems():
            key_wrap, key_inst = key_sol
            gt = self.ground_true.get(key_inst, None)
            if gt is not None:
                for key_metric, metric in self.metrics.iteritems():
                    self.score(key_wrap, key_inst, solution, key_metric, metric, gt)
                continue

            gt = self.solutions.get((gt_name, key_inst), None)
            if gt is not None:
                if key_wrap != gt_name:
                    for key_metric, metric in self.metrics.iteritems():
                        self.score(key_wrap, key_inst, solution, key_metric, metric, gt)
                    continue
            if self.verbose:
                print "{} was not scored because there is not ground truth available".format(key_inst)

    def save_test(self, output_file):
        import pickle
        with open('{}.pkl'.format(output_file), 'wb') as f:
            pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)

    def remove_wrapper(self, name):
        if name in self.wrappers:
            del self.wrappers[name]
        else:
            print 'No wrapper named {} found'.format(name)

    def clear_wrappers(self):
        self.wrappers.clear()

    def remove_solution(self, name):
        if name in self.solutions:
            del self.solutions[name]
        else:
            print 'No solution named {} found'.format(name)

    def clear_solutions(self, wrapper=''):
        if wrapper is '':
            self.solutions.clear()
        else:
            if wrapper in self.wrappers:
                names = self.filter(self.solutions.keys(), 0, wrapper)
                for name in names:
                    self.remove_solution(name)
            else:
                print 'No wrapper named {} found'.format(wrapper)

    def remove_scores(self, name):
        if name in self.scores:
            del self.solutions[name]
        else:
            print 'No score named {} found'.format(name)

    def clear_scores(self, wrapper='', metric=''):
        if wrapper is '' and metric is '':
            self.solutions.clear()
        else:
            names = self.solutions.keys()
            if wrapper in self.wrappers:
                names = self.filter(names, 0, wrapper)

            if metric in self.metrics:
                names = self.filter(names, 2, metric)

            for name in names:
                self.remove_solution(name)

    @staticmethod
    def load_test(input_file):
        import pickle
        with open('{}.pkl'.format(input_file), 'rb') as f:
            return pickle.load(f)

    @staticmethod
    def assert_wrappers(wrappers):
        types = set()
        for wrapper in wrappers:
            if not isinstance(wrapper, QaWrapper):
                raise EssentiaException('Input should be wrappers')
            types.add(wrapper.test_type)
        if len(types) > 1:
            raise EssentiaException("Wrappers type don't match")

    @staticmethod
    def assert_test_type(test_type):
        if test_type.lower() not in test_types:
            raise EssentiaException("'{}' is not supported among the test types ({})"
                                    .format(test_type, test_types))

    @staticmethod
    def assert_metrics(metrics):
        for metric in metrics:
            if not isinstance(metric, QaMetric):
                raise EssentiaException('Metrics should be wrapped in a QaMetric object')

    @staticmethod
    def filter(tuples, pos, patt):
        aux = [idx for idx, x in enumerate(tuples) if x[pos] == patt]
        filtered = []

        for idx in aux:
            filtered.append(tuples[idx])
        return filtered

class QaWrapper:
    test_type = ''
    ground_true = False
    name = ''

    def __init__(self, test_type='values', ground_true=False):
        QaTest.assert_test_type(test_type)
        self.test_type = test_type
        self.ground_true = ground_true
        self.name = self.__class__.__name__
        pass

    def compute(self):
        """
        This method should be overloaded in the sibling class.
        :return:
        """
        pass

    def compute_and_time(self, *args):
        import timeit
        import functools

        t = timeit.Timer(functools.partial(self.compute, *args))
        secs = t.timeit(1)

        solution = self.compute(*args)

        return solution, secs

class QaMetric:
    def __init__(self):
        pass

    def score(self):
        """
        This method should be overloaded in the sibling class.
        :return:
        """
        pass
<|endoftext|>


<commit_before>import torch
import torch.nn as nn
from torch.nn import CrossEntropyLoss
from torch.optim import SGD

from avalanche.benchmarks import Experience
from avalanche.benchmarks.classic import SplitMNIST
from avalanche.benchmarks.generators.benchmark_generators import \
    data_incremental_benchmark
from avalanche.benchmarks.utils import AvalancheSubset
from avalanche.evaluation.metrics import \
    accuracy_metrics, \
    loss_metrics
from avalanche.logging import InteractiveLogger
from avalanche.training.plugins import EvaluationPlugin
from avalanche.training.strategies import GSS_greedy
"""
This example the strategy GSS_greedy on Split MNIST.
The final accuracy is around 82.6% (std 2.9)
as stated in the original paper: https://arxiv.org/abs/1903.08671
"""

class FlattenP(nn.Module):
    '''A nn-module to flatten a multi-dimensional tensor to 2-dim tensor.'''

    def forward(self, x):
        batch_size = x.size(0)   # first dimenstion should be batch-dimension.
        return x.view(batch_size, -1)

    def __repr__(self):
        tmpstr = self.__class__.__name__ + '()'
        return tmpstr

class MLP(nn.Module):
    def __init__(self, sizes, bias=True):
        super(MLP, self).__init__()
        layers = []

        for i in range(0, len(sizes) - 1):

            if i < (len(sizes)-2):
                layers.append(nn.Linear(sizes[i], sizes[i + 1]))

                layers.append(nn.ReLU())
            else:
                layers.append(nn.Linear(sizes[i], sizes[i + 1], bias=bias))

        self.net = nn.Sequential(FlattenP(), *layers)

    def forward(self, x):
        return self.net(x)

def shrinking_experience_size_split_strategy(
        experience: Experience):

    experience_size = 1000

    exp_dataset = experience.dataset
    exp_indices = list(range(len(exp_dataset)))

    result_datasets = []

    exp_indices = \
        torch.as_tensor(exp_indices)[
            torch.randperm(len(exp_indices))
        ].tolist()

    result_datasets.append(AvalancheSubset(
        exp_dataset, indices=exp_indices[0:experience_size]))

    return result_datasets

def setup_mnist():

    scenario = data_incremental_benchmark(SplitMNIST(
        n_experiences=5, seed=1), experience_size=0,
        custom_split_strategy=shrinking_experience_size_split_strategy)
    n_inputs = 784
    nh = 100
    nl = 2
    n_outputs = 10
    model = MLP([n_inputs] + [nh] * nl + [n_outputs])

    return model, scenario

if __name__ == "__main__":

    dev = "cuda:0"

    device = torch.device(dev)

    model, scenario = setup_mnist()

    eval_plugin = EvaluationPlugin(
        accuracy_metrics(epoch=True, experience=True, stream=True),
        loss_metrics(stream=True), loggers=[InteractiveLogger()])

    # _____________________________Strategy

    optimizer = SGD(model.parameters(), lr=0.05)
    strategy = GSS_greedy(model, optimizer, criterion=CrossEntropyLoss(),
                          train_mb_size=10, mem_strength=10,
                          input_size=[1, 28, 28], train_epochs=3,
                          eval_mb_size=10, mem_size=300,
                          evaluator=eval_plugin)

    # ___________________________________________train
    for experience in scenario.train_stream:
        print(">Experience ", experience.current_experience)

        res = strategy.train(experience)

        res = strategy.eval(scenario.test_stream)
<commit_msg>MOVE gss script to reproducibility repo<commit_after><|endoftext|>


<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.util.common import *

if sys.version >= '3':
    long = int
    unicode = str

class Preprocessing(JavaValue):
    """
    Preprocessing defines data transform action during feature preprocessing. Python wrapper for
    the scala Preprocessing
    """
    def __init__(self, bigdl_type="float", *args):
        self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), *args)

class ChainedPreprocessing(Preprocessing):
    """
    chains two Preprocessing together. The output type of the first
    Preprocessing should be the same with the input type of the second Preprocessing.
    """
    def __init__(self, transformers, bigdl_type="float"):
        for transfomer in transformers:
            assert transfomer.__class__.__bases__[0].__name__ in [
                "Preprocessing"], \
                str(transfomer) + " should be subclass of Preprocessing"
        super(ChainedPreprocessing, self).__init__(bigdl_type, transformers)

class ScalarToTensor(Preprocessing):
    """
    a Preprocessing that converts a number to a Tensor.
    """
    def __init__(self, bigdl_type="float"):
        super(ScalarToTensor, self).__init__(bigdl_type)

class SeqToTensor(Preprocessing):
    """
    a Transformer that converts an Array[_] or Seq[_] to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size, bigdl_type="float"):
        super(SeqToTensor, self).__init__(bigdl_type, size)

class ArrayToTensor(Preprocessing):
    """
    a Transformer that converts an Array[_] to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size, bigdl_type="float"):
        super(ArrayToTensor, self).__init__(bigdl_type, size)

class MLlibVectorToTensor(Preprocessing):
    """
    a Transformer that converts MLlib Vector to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size, bigdl_type="float"):
        super(MLlibVectorToTensor, self).__init__(bigdl_type, size)

class ImageFeatureToTensor(Preprocessing):
    """
    a Transformer that convert ImageFeature to a Tensor.
    """
    def __init__(self, bigdl_type="float"):
        super(ImageFeatureToTensor, self).__init__(bigdl_type)

class RowToImageFeature(Preprocessing):
    """
    a Transformer that converts a Spark Row to a BigDL ImageFeature.
    """
    def __init__(self, bigdl_type="float"):
        super(RowToImageFeature, self).__init__(bigdl_type)

class FeatureLabelPreprocessing(Preprocessing):
    """
    construct a Transformer that convert (Feature, Label) tuple to a Sample.
    The returned Transformer is robust for the case label = null, in which the
    Sample is derived from Feature only.
    :param feature_transformer transformer for feature, transform F to Tensor[T]
    :param label_transformer transformer for label, transform L to Tensor[T]
    """
    def __init__(self, feature_transformer, label_transformer, bigdl_type="float"):
        super(FeatureLabelPreprocessing, self).__init__(bigdl_type,
                                                        feature_transformer, label_transformer)

class TensorToSample(Preprocessing):
    """
     a Transformer that converts Tensor to Sample.
    """
    def __init__(self, bigdl_type="float"):
        super(TensorToSample, self).__init__(bigdl_type)

class FeatureToTupleAdapter(Preprocessing):
    def __init__(self, sample_transformer, bigdl_type="float"):
        super(FeatureToTupleAdapter, self).__init__(bigdl_type, sample_transformer)

class BigDLAdapter(Preprocessing):
    def __init__(self, bigdl_transformer, bigdl_type="float"):
        super(BigDLAdapter, self).__init__(bigdl_type, bigdl_transformer)
<commit_msg>New NNFrames interface (#162)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.util.common import *

if sys.version >= '3':
    long = int
    unicode = str

class Preprocessing(JavaValue):
    """
    Preprocessing defines data transform action during feature preprocessing. Python wrapper for
    the scala Preprocessing
    """
    def __init__(self, bigdl_type="float", *args):
        self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self), *args)

class ChainedPreprocessing(Preprocessing):
    """
    chains two Preprocessing together. The output type of the first
    Preprocessing should be the same with the input type of the second Preprocessing.
    """
    def __init__(self, transformers, bigdl_type="float"):
        for transfomer in transformers:
            assert transfomer.__class__.__bases__[0].__name__ in [
                "Preprocessing"], \
                str(transfomer) + " should be subclass of Preprocessing"
        super(ChainedPreprocessing, self).__init__(bigdl_type, transformers)

class ScalarToTensor(Preprocessing):
    """
    a Preprocessing that converts a number to a Tensor.
    """
    def __init__(self, bigdl_type="float"):
        super(ScalarToTensor, self).__init__(bigdl_type)

class SeqToTensor(Preprocessing):
    """
    a Transformer that converts an Array[_] or Seq[_] to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size=[], bigdl_type="float"):
        super(SeqToTensor, self).__init__(bigdl_type, size)

class ArrayToTensor(Preprocessing):
    """
    a Transformer that converts an Array[_] to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size, bigdl_type="float"):
        super(ArrayToTensor, self).__init__(bigdl_type, size)

class MLlibVectorToTensor(Preprocessing):
    """
    a Transformer that converts MLlib Vector to a Tensor.
    :param size dimensions of target Tensor.
    """
    def __init__(self, size, bigdl_type="float"):
        super(MLlibVectorToTensor, self).__init__(bigdl_type, size)

class ImageFeatureToTensor(Preprocessing):
    """
    a Transformer that convert ImageFeature to a Tensor.
    """
    def __init__(self, bigdl_type="float"):
        super(ImageFeatureToTensor, self).__init__(bigdl_type)

class RowToImageFeature(Preprocessing):
    """
    a Transformer that converts a Spark Row to a BigDL ImageFeature.
    """
    def __init__(self, bigdl_type="float"):
        super(RowToImageFeature, self).__init__(bigdl_type)

class FeatureLabelPreprocessing(Preprocessing):
    """
    construct a Transformer that convert (Feature, Label) tuple to a Sample.
    The returned Transformer is robust for the case label = null, in which the
    Sample is derived from Feature only.
    :param feature_transformer transformer for feature, transform F to Tensor[T]
    :param label_transformer transformer for label, transform L to Tensor[T]
    """
    def __init__(self, feature_transformer, label_transformer, bigdl_type="float"):
        super(FeatureLabelPreprocessing, self).__init__(bigdl_type,
                                                        feature_transformer, label_transformer)

class TensorToSample(Preprocessing):
    """
     a Transformer that converts Tensor to Sample.
    """
    def __init__(self, bigdl_type="float"):
        super(TensorToSample, self).__init__(bigdl_type)

class FeatureToTupleAdapter(Preprocessing):
    def __init__(self, sample_transformer, bigdl_type="float"):
        super(FeatureToTupleAdapter, self).__init__(bigdl_type, sample_transformer)

class BigDLAdapter(Preprocessing):
    def __init__(self, bigdl_transformer, bigdl_type="float"):
        super(BigDLAdapter, self).__init__(bigdl_type, bigdl_transformer)

class ToTuple(Preprocessing):
    """
     a Transformer that converts Feature to (Feature, None).
    """
    def __init__(self, bigdl_type="float"):
        super(ToTuple, self).__init__(bigdl_type)
<|endoftext|>