<commit_before># -*- coding: utf-8 -*-

#...............................licence...........................................
#
#     (C) Copyright 2008 Telefonica Investigacion y Desarrollo
#     S.A.Unipersonal (Telefonica I+D)
#
#     This file is part of Morfeo EzWeb Platform.
#
#     Morfeo EzWeb Platform is free software: you can redistribute it and/or modify
#     it under the terms of the GNU Affero General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     Morfeo EzWeb Platform is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the GNU Affero General Public License
#     along with Morfeo EzWeb Platform.  If not, see <http://www.gnu.org/licenses/>.
#
#     Info about members and contributors of the MORFEO project
#     is available at
#
#     http://morfeo-project.org
#
#...............................licence...........................................#

#
import time
import os
from urlparse import urljoin

from django.conf import settings
from django.core.cache import cache
from django.db import IntegrityError
from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseServerError, HttpResponseBadRequest
from django.shortcuts import get_object_or_404
from django.utils.encoding import smart_str
from django.utils.translation import ugettext as _
from django.views.static import serve

from commons import http_utils
from commons.authentication import Http403
from commons.cache import no_cache, patch_cache_headers
from commons.utils import get_xml_error, json_encode
from commons.get_data import get_widget_data
from commons.logs_exception import TracedServerError
from commons.resource import Resource

from wirecloud.commons.utils.http import get_absolute_reverse_url, get_current_domain
from wirecloud.commons.utils.template import TemplateParseException, TemplateParser
from wirecloud.commons.utils.transaction import commit_on_http_success
from wirecloud.platform.iwidget.utils import deleteIWidget
from wirecloud.platform.models import Widget, IWidget
import wirecloud.platform.widget.utils as showcase_utils
from wirecloud.platform.widget.utils import get_or_create_widget, create_widget_from_template, fix_widget_code
from wirecloud.platform.workspace.utils import create_published_workspace_from_template

def parseAndCreateWidget(request, user, workspaceId, fromWGT):
    try:

        templateURL = None

        if 'url' in request.POST:
            templateURL = request.POST['url']
        elif 'template_uri' in request.POST:
            templateURL = request.POST['template_uri']
        else:
            msg = _("Missing template URL parameter")
            raise Exception(msg)

        if not workspaceId:
            msg = _("Missing workspaceId parameter")
            raise Exception(msg)

        #get or create the Widget
        return get_or_create_widget(templateURL, user, workspaceId, request, fromWGT)

    except TemplateParseException, e:
        msg = _("Error parsing the template: %(msg)s" % {"msg": e.msg})
        raise TracedServerError(e, {'url': templateURL}, request, msg)
    except IntegrityError, e:
        msg = _("Widget already exists")
        raise TracedServerError(e, {'url': templateURL}, request, msg)
    except IOError, e:
        msg = _("The url is not accesible")
        raise TracedServerError(e, {'url': templateURL}, request, msg)
    except Http403:
        raise
    except Exception, e:
        raise TracedServerError(e, {'url': templateURL}, request, str(e))

def deleteWidget(user, name, vendor, version):

    result = {'removedIWidgets': []}

    try:

        widget = Widget.objects.get(name=name, vendor=vendor, version=version)

        # TODO
        # Remove all iwidget that matches this Widget Resource
        iwidgets = IWidget.objects.filter(widget=widget)
        for iwidget in iwidgets:
            result['removedIWidgets'].append(iwidget.id)
            deleteIWidget(iwidget, user)

        widget.delete()

    except Widget.DoesNotExist:
        pass

    return result

class WidgetCollection(Resource):

    @no_cache
    def read(self, request):

        widgets = Widget.objects.filter(users=request.user)

        data_list = [get_widget_data(widget, request) for widget in widgets]
        return HttpResponse(json_encode(data_list), mimetype='application/json; charset=UTF-8')

    @commit_on_http_success
    def create(self, request):

        if 'workspaceId' not in request.POST:
            msg = _("Missing workspaceId parameter")
            json = json_encode({"message": msg, "result": "error"})
            return HttpResponseServerError(json, mimetype='application/json; charset=UTF-8')

        #create the widget
        widget = parseAndCreateWidget(request, request.user, request.POST['workspaceId'], request.POST.get('packaged', False) == 'true')

        return HttpResponse(json_encode(get_widget_data(widget, request)), mimetype='application/json; charset=UTF-8')

class Showcase(Resource):

    @commit_on_http_success
    def create(self, request):

        if 'url' not in request.POST:
            return HttpResponseBadRequest()

        url = request.POST['url']
        template_content = http_utils.download_http_content(url, user=request.user)
        template = TemplateParser(template_content, base=url)

        if template.get_resource_type() == 'widget':
            create_widget_from_template(template, request.user, request)
        else:
            create_published_workspace_from_template(template, request.user)

        return HttpResponse(status=201)

class WidgetEntry(Resource):

    @no_cache
    def read(self, request, vendor, name, version):
        widget = get_object_or_404(Widget, users=request.user, vendor=vendor, name=name, version=version)
        data_fields = get_widget_data(widget, request)
        return HttpResponse(json_encode(data_fields), mimetype='application/json; charset=UTF-8')

    def delete(self, request, vendor, name, version):
        widget = get_object_or_404(Widget, users=request.user, vendor=vendor, name=name, version=version)
        widget.delete()
        return HttpResponse('ok')

class WidgetCodeEntry(Resource):

    def read(self, request, vendor, name, version):

        widget = get_object_or_404(Widget, vendor=vendor, name=name, version=version, users=request.user)

        # check if the xhtml code has been cached
        if widget.xhtml.cacheable:

            cache_key = widget.xhtml.get_cache_key(get_current_domain(request))
            cache_entry = cache.get(cache_key)
            if cache_entry is not None:
                response = HttpResponse(cache_entry['code'], mimetype='%s; charset=UTF-8' % cache_entry['content_type'])
                patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
                return response

        # process xhtml
        xhtml = widget.xhtml

        content_type = xhtml.content_type
        if not content_type:
            content_type = 'text/html'

        force_base = False
        base_url = xhtml.url
        if not base_url.startswith(('http://', 'https://')):
            base_url = get_absolute_reverse_url('wirecloud_showcase.media', args=(base_url.split('/', 4)), request=request)
            force_base = True

        code = xhtml.code
        if not xhtml.cacheable or code == '':
            try:
                if xhtml.url.startswith(('http://', 'https://')):
                    code = http_utils.download_http_content(urljoin(base_url, xhtml.url), user=request.user)
                else:
                    code = http_utils.download_http_content('file://' + os.path.join(showcase_utils.wgt_deployer.root_dir, xhtml.url), user=request.user)

            except Exception, e:
                # FIXME: Send the error or use the cached original code?
                msg = _("XHTML code is not accessible: %(errorMsg)s") % {'errorMsg': e.message}
                return HttpResponse(get_xml_error(msg), mimetype='application/xml; charset=UTF-8')

        if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
            xhtml.code = code
            xhtml.code_timestamp = time.time() * 1000
            xhtml.save()
        elif not xhtml.cacheable and xhtml.code != '':
            xhtml.code = ''
            xhtml.code_timestamp = None
            xhtml.save()

        code = fix_widget_code(code, base_url, content_type, request, force_base=force_base)
        if xhtml.cacheable:
            cache_timeout = 31536000  # 1 year
            cache_entry = {
                'code': code,
                'content_type': content_type,
                'timestamp': xhtml.code_timestamp,
                'timeout': cache_timeout,
            }
            cache.set(cache_key, cache_entry, cache_timeout)
        else:
            cache_timeout = 0

        response = HttpResponse(code, mimetype='%s; charset=UTF-8' % content_type)
        patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
        return response

def serve_showcase_media(request, vendor, name, version, file_path):

    if request.method != 'GET':
        return HttpResponseNotAllowed(('GET',))

    base_dir = showcase_utils.wgt_deployer.get_base_dir(vendor, name, version)
    local_path = os.path.join(base_dir, file_path)

    if not os.path.isfile(local_path):
        return HttpResponse(status=404)

    if not getattr(settings, 'USE_XSENDFILE', False):
        return serve(request, local_path, document_root='/')
    else:
        response = HttpResponse()
        response['X-Sendfile'] = smart_str(local_path)
        return response
<commit_msg>Make wirecloud.platform.widget.views use build_error_response<commit_after># -*- coding: utf-8 -*-

#...............................licence...........................................
#
#     (C) Copyright 2008 Telefonica Investigacion y Desarrollo
#     S.A.Unipersonal (Telefonica I+D)
#
#     This file is part of Morfeo EzWeb Platform.
#
#     Morfeo EzWeb Platform is free software: you can redistribute it and/or modify
#     it under the terms of the GNU Affero General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     Morfeo EzWeb Platform is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the GNU Affero General Public License
#     along with Morfeo EzWeb Platform.  If not, see <http://www.gnu.org/licenses/>.
#
#     Info about members and contributors of the MORFEO project
#     is available at
#
#     http://morfeo-project.org
#
#...............................licence...........................................#

#
import time
import os
from urlparse import urljoin

from django.conf import settings
from django.core.cache import cache
from django.db import IntegrityError
from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseServerError, HttpResponseBadRequest
from django.shortcuts import get_object_or_404
from django.utils.encoding import smart_str
from django.utils.translation import ugettext as _
from django.views.static import serve

from commons import http_utils
from commons.cache import no_cache, patch_cache_headers
from commons.utils import get_xml_error, json_encode
from commons.get_data import get_widget_data
from commons.resource import Resource

from wirecloud.commons.utils.http import build_error_response, get_absolute_reverse_url, get_current_domain
from wirecloud.commons.utils.template import TemplateParseException, TemplateParser
from wirecloud.commons.utils.transaction import commit_on_http_success
from wirecloud.platform.iwidget.utils import deleteIWidget
from wirecloud.platform.models import Widget, IWidget
import wirecloud.platform.widget.utils as showcase_utils
from wirecloud.platform.widget.utils import get_or_create_widget, create_widget_from_template, fix_widget_code
from wirecloud.platform.workspace.utils import create_published_workspace_from_template

def parseAndCreateWidget(request, user, workspaceId, fromWGT):

    templateURL = None

    if 'url' in request.POST:
        templateURL = request.POST['url']
    elif 'template_uri' in request.POST:
        templateURL = request.POST['template_uri']
    else:
        msg = _("Missing template URL parameter")
        raise Exception(msg)

    if not workspaceId:
        msg = _("Missing workspaceId parameter")
        raise Exception(msg)

    # get or create the Widget
    return get_or_create_widget(templateURL, user, workspaceId, request, fromWGT)

def deleteWidget(user, name, vendor, version):

    result = {'removedIWidgets': []}

    try:

        widget = Widget.objects.get(name=name, vendor=vendor, version=version)

        # TODO
        # Remove all iwidget that matches this Widget Resource
        iwidgets = IWidget.objects.filter(widget=widget)
        for iwidget in iwidgets:
            result['removedIWidgets'].append(iwidget.id)
            deleteIWidget(iwidget, user)

        widget.delete()

    except Widget.DoesNotExist:
        pass

    return result

class WidgetCollection(Resource):

    @no_cache
    def read(self, request):

        widgets = Widget.objects.filter(users=request.user)

        data_list = [get_widget_data(widget, request) for widget in widgets]
        return HttpResponse(json_encode(data_list), mimetype='application/json; charset=UTF-8')

    @commit_on_http_success
    def create(self, request):

        if 'workspaceId' not in request.POST:
            msg = _("Missing workspaceId parameter")
            json = json_encode({"message": msg, "result": "error"})
            return HttpResponseServerError(json, mimetype='application/json; charset=UTF-8')

        #create the widget
        try:
            widget = parseAndCreateWidget(request, request.user, request.POST['workspaceId'], request.POST.get('packaged', False) == 'true')
        except TemplateParseException, e:
            msg = _("Error parsing the template: %(msg)s" % {"msg": e.msg})
            return build_error_response(request, 400, msg)
        except IntegrityError, e:
            msg = _("Widget already exists")
            return build_error_response(request, 409, msg)
        except IOError, e:
            msg = _("The url is not accesible")
            return build_error_response(request, 502, msg)

        return HttpResponse(json_encode(get_widget_data(widget, request)), mimetype='application/json; charset=UTF-8')

class Showcase(Resource):

    @commit_on_http_success
    def create(self, request):

        if 'url' not in request.POST:
            return HttpResponseBadRequest()

        url = request.POST['url']
        template_content = http_utils.download_http_content(url, user=request.user)
        template = TemplateParser(template_content, base=url)

        if template.get_resource_type() == 'widget':
            create_widget_from_template(template, request.user, request)
        else:
            create_published_workspace_from_template(template, request.user)

        return HttpResponse(status=201)

class WidgetEntry(Resource):

    @no_cache
    def read(self, request, vendor, name, version):
        widget = get_object_or_404(Widget, users=request.user, vendor=vendor, name=name, version=version)
        data_fields = get_widget_data(widget, request)
        return HttpResponse(json_encode(data_fields), mimetype='application/json; charset=UTF-8')

    def delete(self, request, vendor, name, version):
        widget = get_object_or_404(Widget, users=request.user, vendor=vendor, name=name, version=version)
        widget.delete()
        return HttpResponse('ok')

class WidgetCodeEntry(Resource):

    def read(self, request, vendor, name, version):

        widget = get_object_or_404(Widget, vendor=vendor, name=name, version=version, users=request.user)

        # check if the xhtml code has been cached
        if widget.xhtml.cacheable:

            cache_key = widget.xhtml.get_cache_key(get_current_domain(request))
            cache_entry = cache.get(cache_key)
            if cache_entry is not None:
                response = HttpResponse(cache_entry['code'], mimetype='%s; charset=UTF-8' % cache_entry['content_type'])
                patch_cache_headers(response, cache_entry['timestamp'], cache_entry['timeout'])
                return response

        # process xhtml
        xhtml = widget.xhtml

        content_type = xhtml.content_type
        if not content_type:
            content_type = 'text/html'

        force_base = False
        base_url = xhtml.url
        if not base_url.startswith(('http://', 'https://')):
            base_url = get_absolute_reverse_url('wirecloud_showcase.media', args=(base_url.split('/', 4)), request=request)
            force_base = True

        code = xhtml.code
        if not xhtml.cacheable or code == '':
            try:
                if xhtml.url.startswith(('http://', 'https://')):
                    code = http_utils.download_http_content(urljoin(base_url, xhtml.url), user=request.user)
                else:
                    code = http_utils.download_http_content('file://' + os.path.join(showcase_utils.wgt_deployer.root_dir, xhtml.url), user=request.user)

            except Exception, e:
                # FIXME: Send the error or use the cached original code?
                msg = _("XHTML code is not accessible: %(errorMsg)s") % {'errorMsg': e.message}
                return HttpResponse(get_xml_error(msg), mimetype='application/xml; charset=UTF-8')

        if xhtml.cacheable and (xhtml.code == '' or xhtml.code_timestamp is None):
            xhtml.code = code
            xhtml.code_timestamp = time.time() * 1000
            xhtml.save()
        elif not xhtml.cacheable and xhtml.code != '':
            xhtml.code = ''
            xhtml.code_timestamp = None
            xhtml.save()

        code = fix_widget_code(code, base_url, content_type, request, force_base=force_base)
        if xhtml.cacheable:
            cache_timeout = 31536000  # 1 year
            cache_entry = {
                'code': code,
                'content_type': content_type,
                'timestamp': xhtml.code_timestamp,
                'timeout': cache_timeout,
            }
            cache.set(cache_key, cache_entry, cache_timeout)
        else:
            cache_timeout = 0

        response = HttpResponse(code, mimetype='%s; charset=UTF-8' % content_type)
        patch_cache_headers(response, xhtml.code_timestamp, cache_timeout)
        return response

def serve_showcase_media(request, vendor, name, version, file_path):

    if request.method != 'GET':
        return HttpResponseNotAllowed(('GET',))

    base_dir = showcase_utils.wgt_deployer.get_base_dir(vendor, name, version)
    local_path = os.path.join(base_dir, file_path)

    if not os.path.isfile(local_path):
        return HttpResponse(status=404)

    if not getattr(settings, 'USE_XSENDFILE', False):
        return serve(request, local_path, document_root='/')
    else:
        response = HttpResponse()
        response['X-Sendfile'] = smart_str(local_path)
        return response
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
import logging

# Django
from auth0.v3.management import Auth0
from auth0.v3.management.rest import Auth0Error

from django.conf import settings

# Local
from api.models import (
    User,
)

from api.utils import get_auth0_token

log = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Command to sync database with Auth0."

    def get_auth0(self):
        token = get_auth0_token()
        return Auth0(
            settings.AUTH0_DOMAIN,
            token,
        )

    def get_auth0_users(self):
        auth0 = self.get_auth0()
        lst = []
        more = True
        i = 0
        while more:
            results = auth0.users.list(
                fields=[
                    'user_id',
                ],
                per_page=100,
                page=i,
            )
            payload = [result['user_id'] for result in results['users']]
            lst.extend(payload)
            more = bool(results['users'])
            i += 1
        return lst

    def update_or_create_auth0(self, user):
        auth0 = self.get_auth0()
        email = user.email
        try:
            name = user.person.name
        except ValueError:
            name = None
        try:
            bhs_id = user.person.bhs_id
        except ValueError:
            bhs_id = None
        payload = {
            "connection": "email",
            "email": email,
            "email_verified": True,
            "user_metadata": {
                "name": name
            },
            "app_metadata": {
                "bhs_id": bhs_id
            }
        }
        try:
            account, result = auth0.users.update(user.auth0_id, payload), 'Updated'
        except Auth0Error:
            account, result = auth0.users.create(payload), 'Created'
        return account, result

    def handle(self, *args, **options):
        auth0 = self.get_auth0()
        accounts = self.get_auth0_users()
        for account in accounts:
            try:
                user = User.objects.get(
                    auth0_id=account,
                )
            except User.DoesNotExist:
                auth0.users.delete(account)
                log.info("Deleted: {0}".format(account))

        users = User.objects.filter(
            is_staff=False,
        )
        for user in users:
            account, response = self.update_or_create_auth0(user)
            log.info("{1}: {0}".format(account['user_id'], response))
<commit_msg>Update user on auth0 create<commit_after>from django.core.management.base import BaseCommand
import logging

# Django
from auth0.v3.management import Auth0
from auth0.v3.management.rest import Auth0Error

from django.conf import settings

# Local
from api.models import (
    User,
)

from api.utils import get_auth0_token

log = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Command to sync database with Auth0."

    def get_auth0(self):
        token = get_auth0_token()
        return Auth0(
            settings.AUTH0_DOMAIN,
            token,
        )

    def get_auth0_users(self):
        auth0 = self.get_auth0()
        lst = []
        more = True
        i = 0
        while more:
            results = auth0.users.list(
                fields=[
                    'user_id',
                ],
                per_page=100,
                page=i,
            )
            payload = [result['user_id'] for result in results['users']]
            lst.extend(payload)
            more = bool(results['users'])
            i += 1
        return lst

    def update_or_create_auth0(self, user):
        auth0 = self.get_auth0()
        email = user.email
        try:
            name = user.person.name
        except ValueError:
            name = None
        try:
            bhs_id = user.person.bhs_id
        except ValueError:
            bhs_id = None
        payload = {
            "connection": "email",
            "email": email,
            "email_verified": True,
            "user_metadata": {
                "name": name
            },
            "app_metadata": {
                "bhs_id": bhs_id
            }
        }
        try:
            account, result = auth0.users.update(user.auth0_id, payload), 'Updated'
        except Auth0Error:
            account, result = auth0.users.create(payload), 'Created'
        return account, result

    def handle(self, *args, **options):
        auth0 = self.get_auth0()
        accounts = self.get_auth0_users()
        for account in accounts:
            try:
                user = User.objects.get(
                    auth0_id=account,
                )
            except User.DoesNotExist:
                auth0.users.delete(account)
                log.info("Deleted: {0}".format(account))

        users = User.objects.filter(
            is_staff=False,
        )
        for user in users:
            account, response = self.update_or_create_auth0(user)
            if response == 'Created':
                user.auth0_id = account['user_id']
                user.save()
            log.info("{1}: {0}".format(account['user_id'], response))
<|endoftext|>


<commit_before>__author__ = "dbr/Ben"
__version = "0.1"

class _Ddict(dict):
    """Lazy-dict, automatically creates multidimensional dicts
    by having __getitem__ create sub-dict automatically"""
    def __init__(self, default=None):
        self.default = default
    #end __init__

    def __getitem__(self, key):
        if not self.has_key(key):
            self[key] = self.__class__(self.default) # Create sub-instance
        return dict.__getitem__(self, key)
    #end __getitem__
#end _Ddict

# Custom exceptions
class tvdb_error(Exception):pass
class tvdb_shownotfound(Exception):pass
class tvdb_userabort(Exception):pass

class tvdb:
    """
    Create easy-to-use interface to name of season/episode name
    >>> i = tvdb()
    >>> i['showname']['1']['24']['name']
    'Last Episode'
    """
    import urllib
    from BeautifulSoup import BeautifulStoneSoup

    def __init__(self,interactive=False,debug=False):
        self.config={}
        self.config['apikey'] = "0629B785CE550C8D"
        # The following url_ configs are based of the http://www.thetvdb.com API documentation
        self.config['url_mirror'] = "http://www.thetvdb.com/api/%s/mirrors.xml" % (self.config['apikey'])
        self.config['url_getSeries'] = "http://www.thetvdb.com/api/GetSeries.php?seriesname=%s"
        self.config['url_epInfo'] = "http://www.thetvdb.com/api/%s/series/%%s/all/" % (self.config['apikey'])

        self.config['interactive'] = interactive # prompt for correct series if needed

        self.config['debug_enabled'] = debug # show debugging messages
        self.config['debug_tofile'] = False
        self.config['debug_filename'] = "tvdb.log"
        self.config['debug_path'] = '.'

        self.log = self.initLogger() # Setups the logger (self.log.debug() etc)
        self.shows = {} # Holds all show data in shows[show_id] = dict of ep data
        self.corrections = {} # Holds show-name to show_id mapping

        # Config setup. Grab TVDB mirrors
        self.mirrors = self._getMirrors() # TODO: Apply random mirror urls (Minor: Currently 1 mirror)
    #end __init__

    def initLogger(self):
        import os,logging,sys
        logdir = os.path.expanduser( self.config['debug_path'] )
        logpath = os.path.join(logdir,self.config['debug_filename'])

        logger = logging.getLogger("tvdb")
        formatter = logging.Formatter('%(asctime)s) %(levelname)s %(message)s')

        if self.config['debug_tofile']:
            hdlr = logging.FileHandler(logpath)
        else:
            hdlr = logging.StreamHandler(sys.stdout)
        #end if debug_tofile

        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)

        if self.config['debug_enabled']:
            logger.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO)
        return logger
    #end initLogger

    def _getsoupsrc(self,url):
        self.log.debug('Retriving URL %s' % (url.replace(" ","+")))

        url=url.replace(" ","+")
        try:
            src=self.urllib.urlopen(url).read()
        except IOError,errormsg:
            raise tvdb_error("Could not connect to server: %s\n" % (errormsg))
        #end try
        soup=self.BeautifulStoneSoup(src)
        return soup
    #end _getsoupsrc

    def _getMirrors(self):
        mirrorSoup=self._getsoupsrc( self.config['url_mirror'] )
        mirrors=[]
        for mirror in mirrorSoup.findAll('mirror'):
            self.log.debug('Found mirror %s' % (mirror))

            mirrors.append(
                mirror.find('mirrorpath').contents[0]
            )
        #end for mirror
        self.log.debug('Found total of %s mirrors' % (len(mirrors)))
        return mirrors
    #end _getMirrors

    def _getSeries(self,series):
        seriesSoup = self._getsoupsrc( self.config['url_getSeries'] % (series) )
        allSeries=[]
        for series in seriesSoup.findAll('series'):
            cur_name = series.find('seriesname').contents[0]
            cur_sid = series.find('id').contents[0]
            self.log.debug('Found series %s (id: %s)' % (cur_name,cur_sid))
            allSeries.append( {'sid':cur_sid, 'name':cur_name} )
        #end for series

        if len(allSeries) == 0:
            self.log.debug('Series result returned zero')
            raise tvdb_shownotfound("Show-name search returned zero results")

        if self.config['interactive']:
            self.log.debug('Interactivily selecting show')
            for i in range(len(allSeries[:6])):
                i_show = i + 1 # Start at more human readable number 1 (not zero)
                self.log.debug( 'Showing allSeries[%s] = %s)' % (i_show,allSeries[i]) )
                print "%s -> %s (tvdb id: %s)" % (i_show,allSeries[i]['name'],allSeries[i]['sid'])
            print "Enter choice (first number):"
            ans=raw_input()
            self.log.debug( 'Got choice of: %s' % (ans))
            try:
                selected_id = int(ans) - 1 # The human entered 1 as first result, not zero
                self.log.debug( 'Trying to return ID: %d' % (selected_id))
                return allSeries[ selected_id ]
            except ValueError: # Input was not number
                if ans == "q":
                    self.log.debug('Got quit command (q)')
                    raise tvdb_userabort("User aborted")
                else:
                    self.log.debug('Unknown keypress %s' % (ans))
                    raise tvdb_userabort("Invalid keypress") # TODO: Better UI
            #end for k,v
        else:
            self.log.debug('Auto-selecting first search result')
            return allSeries[0]
    #end _getSeries

    def _getEps(self,sid):
        self.log.debug('Getting all episodes of %s' % (sid))
        epsSoup=self._getsoupsrc( self.config['url_epInfo']% (sid) )
        for ep in epsSoup.findAll('episode'):
            ep_no = int( ep.find('episodenumber').contents[0] )
            seas_no = int( ep.find('seasonnumber').contents[0] )
            ep_name = str( ep.find('episodename').contents[0] )

            self.shows[sid][seas_no][ep_no] = {'name':ep_name}
        #end for ep
    #end _geEps

    def _nameToSid(self,name):
        """
        Takes show name, returns the correct series ID (if the show has
        already been grabbed), or grabs all episodes and returns
        the correct SID.
        """
        if self.corrections.has_key(name):
            self.log.debug('Correcting %s to %s' % (name,self.corrections[name]) )
            sid = self.corrections[name]
        else:
            self.log.debug('Getting show %s' % (name))
            selected_series = self._getSeries( name )
            sname, sid = selected_series['name'], selected_series['sid']
            self.log.debug( "Got %s, sid %s" % (sname,sid) )
            self.shows[sid] = _Ddict(dict)
            self.shows[sid]['showname'] = sname
            self.corrections[name] = sid
            self._getEps( sid )
        #end if self.corrections.has_key
        return sid
    #end _nameToSid

    def __getitem__(self,key):
        """
        Handles tvdb_instance['showname'] calls.
        The dict index should be the show name
        """
        key=key.lower() # make key lower case
        sid = self._nameToSid(key)
        self.log.debug('Got series id %s' % (sid))
        return dict.__getitem__(self.shows, sid)
    #end __getitem__

    def __setitem__(self,key,value):
        self.log.debug('Setting %s = %s' % (key,value))
        self.shows[key] = value
    #end __getitem__
    def __str__(self):
        return str(self.shows) #TODO: Improve this
    #end __str__
#end tvdb

if __name__ == '__main__':
    x=tvdb(interactive=True,debug=True)
    print x['lost'][1][4]
    print x['Lost'][1][4]
<commit_msg>Added basic Cache class to tvdb_api<commit_after>__author__ = "dbr/Ben"
__version = "0.1"

class _Ddict(dict):
    """Lazy-dict, automatically creates multidimensional dicts
    by having __getitem__ create sub-dict automatically"""
    def __init__(self, default=None):
        self.default = default
    #end __init__

    def __getitem__(self, key):
        if not self.has_key(key):
            self[key] = self.__class__(self.default) # Create sub-instance
        return dict.__getitem__(self, key)
    #end __getitem__
#end _Ddict

class Cache:
    import os
    import tempfile
    import urllib
    try:
        import sha1 as hasher
    except ImportError:
        import md5 as hasher

    def __init__(self,prefix="tvdb_api"):
        self.prefix = prefix
        tmp = self.tempfile.gettempdir()
        tmppath = self.os.path.join(tmp, prefix)
        if not self.os.path.isdir(tmppath):
            self.os.mkdir(tmppath)
        self.tmp = tmppath
    #end __init__

    def getCachePath(self,url):
        cache_name = self.hasher.new(url).hexdigest()
        cache_path = self.os.path.join(self.tmp, cache_name)
        return cache_path
    #end getUrl

    def checkCache(self,url):
        path = self.getCachePath(url)
        if self.os.path.isfile(path):
            return path
        else:
            return False
    #end checkCache

    def getUrl(self,url):
        cacheExists = self.checkCache(url)
        if cacheExists:
            f=open(cacheExists)
            dat = f.read()
            f.close()
            return dat
        else:
            path = self.getCachePath(url)
            dat = self.urllib.urlopen(url).read()
            f=open(path,"w+")
            f.write(dat)
            f.close()
            return dat
        #end if cacheExists
    #end getUrl
#end Cache

# Custom exceptions
class tvdb_error(Exception):pass
class tvdb_shownotfound(Exception):pass
class tvdb_userabort(Exception):pass

class tvdb:
    """
    Create easy-to-use interface to name of season/episode name
    >>> i = tvdb()
    >>> i['showname']['1']['24']['name']
    'Last Episode'
    """
    import urllib
    from BeautifulSoup import BeautifulStoneSoup

    def __init__(self,interactive=False,debug=False):
        self.config={}
        self.config['apikey'] = "0629B785CE550C8D"
        # The following url_ configs are based of the http://www.thetvdb.com API documentation
        self.config['url_mirror'] = "http://www.thetvdb.com/api/%s/mirrors.xml" % (self.config['apikey'])
        self.config['url_getSeries'] = "http://www.thetvdb.com/api/GetSeries.php?seriesname=%s"
        self.config['url_epInfo'] = "http://www.thetvdb.com/api/%s/series/%%s/all/" % (self.config['apikey'])

        self.config['interactive'] = interactive # prompt for correct series if needed

        self.config['debug_enabled'] = debug # show debugging messages
        self.config['debug_tofile'] = False
        self.config['debug_filename'] = "tvdb.log"
        self.config['debug_path'] = '.'

        self.log = self.initLogger() # Setups the logger (self.log.debug() etc)
        self.shows = {} # Holds all show data in shows[show_id] = dict of ep data
        self.corrections = {} # Holds show-name to show_id mapping

        # Config setup. Grab TVDB mirrors
        self.mirrors = self._getMirrors() # TODO: Apply random mirror urls (Minor: Currently 1 mirror)
    #end __init__

    def initLogger(self):
        import os,logging,sys
        logdir = os.path.expanduser( self.config['debug_path'] )
        logpath = os.path.join(logdir,self.config['debug_filename'])

        logger = logging.getLogger("tvdb")
        formatter = logging.Formatter('%(asctime)s) %(levelname)s %(message)s')

        if self.config['debug_tofile']:
            hdlr = logging.FileHandler(logpath)
        else:
            hdlr = logging.StreamHandler(sys.stdout)
        #end if debug_tofile

        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)

        if self.config['debug_enabled']:
            logger.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO)
        return logger
    #end initLogger

    def _getsoupsrc(self,url):
        self.log.debug('Retriving URL %s' % (url.replace(" ","+")))

        url=url.replace(" ","+")
        try:
            src=self.urllib.urlopen(url).read()
        except IOError,errormsg:
            raise tvdb_error("Could not connect to server: %s\n" % (errormsg))
        #end try
        soup=self.BeautifulStoneSoup(src)
        return soup
    #end _getsoupsrc

    def _getMirrors(self):
        mirrorSoup=self._getsoupsrc( self.config['url_mirror'] )
        mirrors=[]
        for mirror in mirrorSoup.findAll('mirror'):
            self.log.debug('Found mirror %s' % (mirror))

            mirrors.append(
                mirror.find('mirrorpath').contents[0]
            )
        #end for mirror
        self.log.debug('Found total of %s mirrors' % (len(mirrors)))
        return mirrors
    #end _getMirrors

    def _getSeries(self,series):
        seriesSoup = self._getsoupsrc( self.config['url_getSeries'] % (series) )
        allSeries=[]
        for series in seriesSoup.findAll('series'):
            cur_name = series.find('seriesname').contents[0]
            cur_sid = series.find('id').contents[0]
            self.log.debug('Found series %s (id: %s)' % (cur_name,cur_sid))
            allSeries.append( {'sid':cur_sid, 'name':cur_name} )
        #end for series

        if len(allSeries) == 0:
            self.log.debug('Series result returned zero')
            raise tvdb_shownotfound("Show-name search returned zero results")

        if self.config['interactive']:
            self.log.debug('Interactivily selecting show')
            for i in range(len(allSeries[:6])):
                i_show = i + 1 # Start at more human readable number 1 (not zero)
                self.log.debug( 'Showing allSeries[%s] = %s)' % (i_show,allSeries[i]) )
                print "%s -> %s (tvdb id: %s)" % (i_show,allSeries[i]['name'],allSeries[i]['sid'])
            print "Enter choice (first number):"
            ans=raw_input()
            self.log.debug( 'Got choice of: %s' % (ans))
            try:
                selected_id = int(ans) - 1 # The human entered 1 as first result, not zero
                self.log.debug( 'Trying to return ID: %d' % (selected_id))
                return allSeries[ selected_id ]
            except ValueError: # Input was not number
                if ans == "q":
                    self.log.debug('Got quit command (q)')
                    raise tvdb_userabort("User aborted")
                else:
                    self.log.debug('Unknown keypress %s' % (ans))
                    raise tvdb_userabort("Invalid keypress") # TODO: Better UI
            #end for k,v
        else:
            self.log.debug('Auto-selecting first search result')
            return allSeries[0]
    #end _getSeries

    def _getEps(self,sid):
        self.log.debug('Getting all episodes of %s' % (sid))
        epsSoup=self._getsoupsrc( self.config['url_epInfo']% (sid) )
        for ep in epsSoup.findAll('episode'):
            ep_no = int( ep.find('episodenumber').contents[0] )
            seas_no = int( ep.find('seasonnumber').contents[0] )
            ep_name = str( ep.find('episodename').contents[0] )

            self.shows[sid][seas_no][ep_no] = {'name':ep_name}
        #end for ep
    #end _geEps

    def _nameToSid(self,name):
        """
        Takes show name, returns the correct series ID (if the show has
        already been grabbed), or grabs all episodes and returns
        the correct SID.
        """
        if self.corrections.has_key(name):
            self.log.debug('Correcting %s to %s' % (name,self.corrections[name]) )
            sid = self.corrections[name]
        else:
            self.log.debug('Getting show %s' % (name))
            selected_series = self._getSeries( name )
            sname, sid = selected_series['name'], selected_series['sid']
            self.log.debug( "Got %s, sid %s" % (sname,sid) )
            self.shows[sid] = _Ddict(dict)
            self.shows[sid]['showname'] = sname
            self.corrections[name] = sid
            self._getEps( sid )
        #end if self.corrections.has_key
        return sid
    #end _nameToSid

    def __getitem__(self,key):
        """
        Handles tvdb_instance['showname'] calls.
        The dict index should be the show name
        """
        key=key.lower() # make key lower case
        sid = self._nameToSid(key)
        self.log.debug('Got series id %s' % (sid))
        return dict.__getitem__(self.shows, sid)
    #end __getitem__

    def __setitem__(self,key,value):
        self.log.debug('Setting %s = %s' % (key,value))
        self.shows[key] = value
    #end __getitem__
    def __str__(self):
        return str(self.shows) #TODO: Improve this
    #end __str__
#end tvdb

if __name__ == '__main__':
    x=tvdb(interactive=True,debug=True)
    print x['lost'][1][4]
    print x['Lost'][1][4]
<|endoftext|>


<commit_before><commit_msg>handles error when subsetting cols in phenotype group, but not in study<commit_after><|endoftext|>


<commit_before>import json
import collections

import requests
from django.conf import settings
from django.template.loader import render_to_string
from django.utils.encoding import force_bytes, force_text
from django.utils.safestring import mark_safe
from django.utils.translation import get_language

from six.moves import urllib

RecaptchaResponse = collections.namedtuple(
    'RecaptchaResponse',
    'is_valid, error_code')

class ReCaptcha(object):
    VERIFY_URL = '{scheme}://google.com/recaptcha/api/verify'
    API_SERVER = '//www.google.com/recaptcha/api'
    SUPPORTED_LANGUAGES = ('en', 'nl', 'fr', 'de', 'pt', 'ru', 'es', 'tr')

    site_key = None
    secret_key = None
    nocaptcha = False
    template = 'captcha/widget.html'

    def __init__(self, site_key=None, secret_key=None, use_ssl=False):
        self.site_key = site_key or settings.NOBOT_RECAPTCHA_PUBLIC_KEY
        self.secret_key = secret_key or settings.NOBOT_RECAPTCHA_PRIVATE_KEY
        self.use_ssl = use_ssl or settings.NOBOT_RECAPTCHA_USE_SSL

    def render(self, attrs, error=None):
        if 'lang' not in attrs:
            attrs['lang'] = get_language()[:2]

        args = {
            'k': self.site_key,
            'hl': attrs['lang'],
        }

        if error:
            args['error'] = error

        challenge_url = (
            self.API_SERVER + '/challenge' + urllib.parse.urlencode(args))
        noscript_url = (
            self.API_SERVER + '/noscript' + urllib.parse.urlencode(args))

        return render_to_string(
            self.template,
            {
                'api_server': self.API_SERVER,
                'public_key': self.site_key,
                'lang': attrs['lang'],
                'options': mark_safe(json.dumps(attrs, indent=2)),
                'challenge_url': challenge_url,
                'noscript_url': noscript_url
            }
        )

    def verify(self, challenge, response, remote_ip):
        if not (response and challenge and len(response) and len(challenge)):
            return RecaptchaResponse(
                is_valid=False,
                error_code='incorrect-captcha-sol'
            )

        data = {
            'privatekey': force_bytes(self.secret_key),
            'remoteip':  force_bytes(remote_ip),
            'challenge':  force_bytes(challenge),
            'response':  force_bytes(response),
        }

        verify_url = self.VERIFY_URL.format(
            scheme='https' if self.use_ssl else 'http'
        )

        r = requests.get(verify_url, params=data)

        return_code, error_code = force_text(r.content).splitlines()

        if (str(return_code) == 'true'):
            return RecaptchaResponse(is_valid=True, error_code=None)
        else:
            return RecaptchaResponse(is_valid=False, error_code=error_code)

class HumanCaptcha(ReCaptcha):
    VERIFY_URL = 'https://www.google.com/recaptcha/api/siteverify'
    template = 'captcha/widget_nocaptcha.html'
    nocaptcha = True

    def verify(self, challenge, response, remote_ip):
        data = {
            'secret': force_bytes(self.secret_key),
            'response': force_bytes(response),
            'remoteip': force_bytes(remote_ip)
        }

        verified = requests.get(self.VERIFY_URL, params=data)

        if verified.status_code == 200:
            return RecaptchaResponse(is_valid=True, error_code=None)
        else:
            return RecaptchaResponse(is_valid=False, error_code=None)
<commit_msg>fix pep8<commit_after>import json
import collections

import requests
from django.conf import settings
from django.template.loader import render_to_string
from django.utils.encoding import force_bytes, force_text
from django.utils.safestring import mark_safe
from django.utils.translation import get_language

from six.moves import urllib

RecaptchaResponse = collections.namedtuple(
    'RecaptchaResponse',
    'is_valid, error_code')

class ReCaptcha(object):
    VERIFY_URL = '{scheme}://google.com/recaptcha/api/verify'
    API_SERVER = '//www.google.com/recaptcha/api'
    SUPPORTED_LANGUAGES = ('en', 'nl', 'fr', 'de', 'pt', 'ru', 'es', 'tr')

    site_key = None
    secret_key = None
    nocaptcha = False
    template = 'captcha/widget.html'

    def __init__(self, site_key=None, secret_key=None, use_ssl=False):
        self.site_key = site_key or settings.NOBOT_RECAPTCHA_PUBLIC_KEY
        self.secret_key = secret_key or settings.NOBOT_RECAPTCHA_PRIVATE_KEY
        self.use_ssl = use_ssl or settings.NOBOT_RECAPTCHA_USE_SSL

    def render(self, attrs, error=None):
        if 'lang' not in attrs:
            attrs['lang'] = get_language()[:2]

        args = {
            'k': self.site_key,
            'hl': attrs['lang'],
        }

        if error:
            args['error'] = error

        challenge_url = (
            self.API_SERVER + '/challenge' + urllib.parse.urlencode(args))
        noscript_url = (
            self.API_SERVER + '/noscript' + urllib.parse.urlencode(args))

        return render_to_string(
            self.template,
            {
                'api_server': self.API_SERVER,
                'public_key': self.site_key,
                'lang': attrs['lang'],
                'options': mark_safe(json.dumps(attrs, indent=2)),
                'challenge_url': challenge_url,
                'noscript_url': noscript_url
            }
        )

    def verify(self, challenge, response, remote_ip):
        if not (response and challenge and len(response) and len(challenge)):
            return RecaptchaResponse(
                is_valid=False,
                error_code='incorrect-captcha-sol'
            )

        data = {
            'privatekey': force_bytes(self.secret_key),
            'remoteip': force_bytes(remote_ip),
            'challenge': force_bytes(challenge),
            'response': force_bytes(response),
        }

        verify_url = self.VERIFY_URL.format(
            scheme='https' if self.use_ssl else 'http'
        )

        r = requests.get(verify_url, params=data)

        return_code, error_code = force_text(r.content).splitlines()

        if (str(return_code) == 'true'):
            return RecaptchaResponse(is_valid=True, error_code=None)
        else:
            return RecaptchaResponse(is_valid=False, error_code=error_code)

class HumanCaptcha(ReCaptcha):
    VERIFY_URL = 'https://www.google.com/recaptcha/api/siteverify'
    template = 'captcha/widget_nocaptcha.html'
    nocaptcha = True

    def verify(self, challenge, response, remote_ip):
        data = {
            'secret': force_bytes(self.secret_key),
            'response': force_bytes(response),
            'remoteip': force_bytes(remote_ip)
        }

        verified = requests.get(self.VERIFY_URL, params=data)

        if verified.status_code == 200:
            return RecaptchaResponse(is_valid=True, error_code=None)
        else:
            return RecaptchaResponse(is_valid=False, error_code=None)
<|endoftext|>


<commit_before># Standard Libary
import datetime
import logging

# Django
from django.core.management.base import BaseCommand
from django.utils import timezone

# First-Party
from api.models import Person
from api.models import User
from bhs.models import Human
from bhs.models import Structure
from bhs.models import Role
from bhs.models import Join

log = logging.getLogger('updater')

class Command(BaseCommand):
    help = "Command to sync with BHS database."

    def add_arguments(self, parser):
        parser.add_argument(
            '--days',
            type=int,
            dest='days',
            nargs='?',
            const=1,
            help='Number of days to update.',
        )

        parser.add_argument(
            '--hours',
            type=int,
            dest='hours',
            nargs='?',
            const=1,
            help='Number of hours to update.',
        )

        parser.add_argument(
            '--minutes',
            type=int,
            dest='minutes',
            nargs='?',
            const=1,
            help='Number of hours to update.',
        )

    def handle(self, *args, **options):
        # Set Cursor
        if options['days']:
            cursor = timezone.now() - datetime.timedelta(days=options['days'], hours=1)
        elif options['hours']:
            cursor = timezone.now() - datetime.timedelta(hours=options['hours'], minutes=5)
        elif options['minutes']:
            cursor = timezone.now() - datetime.timedelta(minutes=options['minutes'], seconds=5)
        else:
            cursor = None

        # Sync Persons
        t = Human.objects.update_persons(cursor=cursor)
        self.stdout.write("Queued {0} persons.".format(t))
        t = Human.objects.delete_orphans()
        self.stdout.write("Deleted {0} person orphans.".format(t))

        # Sync Groups
        t = Structure.objects.update_groups(cursor=cursor)
        self.stdout.write("Queued {0} groups.".format(t))
        t = Structure.objects.delete_orphans()
        self.stdout.write("Deleted {0} group orphans.".format(t))

        # Sync Members
        t = Join.objects.update_members()
        self.stdout.write("Queued {0} members.".format(t))

        # Sync Roles
        t = Role.objects.update_chapter_officers(cursor=cursor)
        self.stdout.write("Queued {0} chapter officers.".format(t))
        self.stdout.write("BYPASSED chapter officers.")

        # Sync Users
        t = Person.objects.update_users(cursor=cursor)
        self.stdout.write("Queued {0} users.".format(t))

        # Sync Accounts
        t = User.objects.update_accounts(cursor=cursor)
        self.stdout.write("Queued {0} accounts.".format(t))

        self.stdout.write("Complete.")
<commit_msg>Disable chapter role update<commit_after># Standard Libary
import datetime
import logging

# Django
from django.core.management.base import BaseCommand
from django.utils import timezone

# First-Party
from api.models import Person
from api.models import User
from bhs.models import Human
from bhs.models import Structure
# from bhs.models import Role
from bhs.models import Join

log = logging.getLogger('updater')

class Command(BaseCommand):
    help = "Command to sync with BHS database."

    def add_arguments(self, parser):
        parser.add_argument(
            '--days',
            type=int,
            dest='days',
            nargs='?',
            const=1,
            help='Number of days to update.',
        )

        parser.add_argument(
            '--hours',
            type=int,
            dest='hours',
            nargs='?',
            const=1,
            help='Number of hours to update.',
        )

        parser.add_argument(
            '--minutes',
            type=int,
            dest='minutes',
            nargs='?',
            const=1,
            help='Number of hours to update.',
        )

    def handle(self, *args, **options):
        # Set Cursor
        if options['days']:
            cursor = timezone.now() - datetime.timedelta(days=options['days'], hours=1)
        elif options['hours']:
            cursor = timezone.now() - datetime.timedelta(hours=options['hours'], minutes=5)
        elif options['minutes']:
            cursor = timezone.now() - datetime.timedelta(minutes=options['minutes'], seconds=5)
        else:
            cursor = None

        # Sync Persons
        t = Human.objects.update_persons(cursor=cursor)
        self.stdout.write("Queued {0} persons.".format(t))
        t = Human.objects.delete_orphans()
        self.stdout.write("Deleted {0} person orphans.".format(t))

        # Sync Groups
        t = Structure.objects.update_groups(cursor=cursor)
        self.stdout.write("Queued {0} groups.".format(t))
        t = Structure.objects.delete_orphans()
        self.stdout.write("Deleted {0} group orphans.".format(t))

        # Sync Members
        t = Join.objects.update_members()
        self.stdout.write("Queued {0} members.".format(t))

        # Sync Roles
        # t = Role.objects.update_chapter_officers(cursor=cursor)
        # self.stdout.write("Queued {0} chapter officers.".format(t))
        self.stdout.write("BYPASSED chapter officers.")

        # Sync Users
        t = Person.objects.update_users(cursor=cursor)
        self.stdout.write("Queued {0} users.".format(t))

        # Sync Accounts
        t = User.objects.update_accounts(cursor=cursor)
        self.stdout.write("Queued {0} accounts.".format(t))

        self.stdout.write("Complete.")
<|endoftext|>


<commit_before>import csv
import string
import os
from metadata import getSlug, getMakam, getForm, getUsul, validateAttribute

def readTxtScore(scorefile):
    with open(scorefile, "rb") as f:
        reader = csv.reader(f, delimiter='\t')

        header = next(reader, None)

        index_col = header.index('Sira')
        code_col = header.index('Kod')
        note53_col = header.index('Nota53')
        noteAE_col = header.index('NotaAE')
        comma53_col = header.index('Koma53')
        commaAE_col = header.index('KomaAE')
        numerator_col = header.index('Pay')
        denumerator_col = header.index('Payda')
        duration_col = header.index('Ms')
        lyrics_col = header.index('Soz1')
        offset_col = header.index('Offset')

        score = {'index': [], 'code': [], 'note53': [], 'noteAE': [],
                'comma53': [], 'commaAE': [], 'numerator': [],
                'denumerator': [], 'duration': [], 'lyrics': [],
                'offset': []}
        for row in reader:
            score['index'].append(int(row[index_col]))
            score['code'].append(int(row[code_col]))
            score['note53'].append(row[note53_col])
            score['noteAE'].append(row[noteAE_col])
            score['comma53'].append(int(row[comma53_col]))
            score['commaAE'].append(int(row[commaAE_col]))
            score['numerator'].append(int(row[numerator_col]))
            score['denumerator'].append(int(row[denumerator_col]))
            score['duration'].append(int(row[duration_col]))
            score['lyrics'].append(row[lyrics_col].decode('utf-8'))
            score['offset'].append(float(row[offset_col]))

    # shift offset such that the first note of each measure has an integer offset
    score['offset'].insert(0, 0)
    score['offset'] = score['offset'][:-1]

    # validate
    isScoreValid = validateTxtScore(score, os.path.splitext(os.path.basename(scorefile))[0])

    return score, isScoreValid

def validateTxtScore(score, scorename):
    isScoreValid = True
    for ii in range(0, len(score['index'])):
        if score['duration'][ii] > 0:  # note or rest
            # check rest
            if (-1 in [score['comma53'][ii], score['commaAE'][ii]] or
                any(rs in [score['note53'][ii], score['noteAE'][ii]] for rs in ['Es', 'Sus', ''])):
                # it should be rest, validate
                if not (score['comma53'][ii] == -1 and
                    score['commaAE'][ii] == -1 and
                    score['note53'][ii] == 'Es' and
                    score['noteAE'][ii] == 'Es' and
                    score['code'][ii] == 9):
                    isScoreValid = False
                    print scorename + ' ' + str(score['index'][ii]) + ': Invalid Rest'

    return isScoreValid

def readMu2Score(scorefile):
    # TODO
    pass

def readMu2Header(scorefile, symbtrname=''):
    with open(scorefile, "rb") as f:
        reader = csv.reader(f, delimiter='\t')

        headerRow = [unicode(cell, 'utf-8') for cell in next(reader, None)]

        header = dict()
        for rowtemp in reader:
            row = [unicode(cell, 'utf-8') for cell in rowtemp]
            code = int(row[0])
            if code == 50:
                header['makam'] = {'mu2_name':row[7]}
                header['key_signature'] = row[8].split('/')
            elif code == 51:
                header['usul'] = {'mu2_name':row[7], 'mertebe':int(row[3]),
                    'number_of_pulses':int(row[2])}
            elif code == 52:
                try:
                    header['tempo'] = {'value':int(row[4]), 'unit':'bpm'}
                except ValueError:  # the bpm might be a float for low tempo
                    header['tempo'] = {'value':float(row[4]), 'unit':'bpm'}
                if not int(row[3]) == header['usul']['mertebe']:
                    print '   Mertebe and tempo unit are different!'
            elif code == 56:
                header['usul']['subdivision'] = {'mertebe':int(row[3]),
                    'number_of_pulses':int(row[2])}
            elif code == 57:
                header['form'] = {'mu2_name':row[7]}
            elif code == 58:
                header['composer'] = {'mu2_name':row[7]}
            elif code == 59:
                header['lyricist'] = {'mu2_name':row[7]}
            elif code == 60:
                header['title'] = {'mu2_title':row[7]}
            elif code == 62:
                header['genre'] = 'folk' if row[7] == 'E' else 'classical'
            elif code == 63:
                header['notation'] = row[7]
            elif code in range(50, 64):
                print '   Unparsed code: ' + ' '.join(row)
            else:  # end of header
                break

    # get the metadata
    if not symbtrname:
        symbtrname = os.path.splitext(os.path.basename(scorefile))[0]

    slugs = getSlug(symbtrname)
    header['makam']['symbtr_slug'] = slugs['makam']
    header['form']['symbtr_slug'] = slugs['form']
    header['usul']['symbtr_slug'] = slugs['usul']
    header['title']['symbtr_slug'] = slugs['name']
    header['composer']['symbtr_slug'] = slugs['composer']

    # validate the header content
    makam = getMakam(header['makam']['symbtr_slug'])
    isMakamValid = validateAttribute(header['makam'], makam, symbtrname)

    form = getForm(header['form']['symbtr_slug'])
    isFormValid = validateAttribute(header['form'], form, symbtrname)

    usul = getUsul(header['usul']['symbtr_slug'])
    isUsulValid = validateAttribute(header['usul'], usul, symbtrname)

    isHeaderValid = isMakamValid and isFormValid and isUsulValid

    return header, headerRow, isHeaderValid
<commit_msg>symbtr name is printed if the mertebe and the tempo unit are different in an mu2 file<commit_after>import csv
import string
import os
from metadata import getSlug, getMakam, getForm, getUsul, validateAttribute

def readTxtScore(scorefile):
    with open(scorefile, "rb") as f:
        reader = csv.reader(f, delimiter='\t')

        header = next(reader, None)

        index_col = header.index('Sira')
        code_col = header.index('Kod')
        note53_col = header.index('Nota53')
        noteAE_col = header.index('NotaAE')
        comma53_col = header.index('Koma53')
        commaAE_col = header.index('KomaAE')
        numerator_col = header.index('Pay')
        denumerator_col = header.index('Payda')
        duration_col = header.index('Ms')
        lyrics_col = header.index('Soz1')
        offset_col = header.index('Offset')

        score = {'index': [], 'code': [], 'note53': [], 'noteAE': [],
                'comma53': [], 'commaAE': [], 'numerator': [],
                'denumerator': [], 'duration': [], 'lyrics': [],
                'offset': []}
        for row in reader:
            score['index'].append(int(row[index_col]))
            score['code'].append(int(row[code_col]))
            score['note53'].append(row[note53_col])
            score['noteAE'].append(row[noteAE_col])
            score['comma53'].append(int(row[comma53_col]))
            score['commaAE'].append(int(row[commaAE_col]))
            score['numerator'].append(int(row[numerator_col]))
            score['denumerator'].append(int(row[denumerator_col]))
            score['duration'].append(int(row[duration_col]))
            score['lyrics'].append(row[lyrics_col].decode('utf-8'))
            score['offset'].append(float(row[offset_col]))

    # shift offset such that the first note of each measure has an integer offset
    score['offset'].insert(0, 0)
    score['offset'] = score['offset'][:-1]

    # validate
    isScoreValid = validateTxtScore(score, os.path.splitext(os.path.basename(scorefile))[0])

    return score, isScoreValid

def validateTxtScore(score, scorename):
    isScoreValid = True
    for ii in range(0, len(score['index'])):
        if score['duration'][ii] > 0:  # note or rest
            # check rest
            if (-1 in [score['comma53'][ii], score['commaAE'][ii]] or
                any(rs in [score['note53'][ii], score['noteAE'][ii]] for rs in ['Es', 'Sus', ''])):
                # it should be rest, validate
                if not (score['comma53'][ii] == -1 and
                    score['commaAE'][ii] == -1 and
                    score['note53'][ii] == 'Es' and
                    score['noteAE'][ii] == 'Es' and
                    score['code'][ii] == 9):
                    isScoreValid = False
                    print scorename + ' ' + str(score['index'][ii]) + ': Invalid Rest'

    return isScoreValid

def readMu2Score(scorefile):
    # TODO
    pass

def readMu2Header(scorefile, symbtrname=''):
    if not symbtrname:
        symbtrname = os.path.splitext(os.path.basename(scorefile))[0]

    with open(scorefile, "rb") as f:
        reader = csv.reader(f, delimiter='\t')

        headerRow = [unicode(cell, 'utf-8') for cell in next(reader, None)]

        header = dict()
        for rowtemp in reader:
            row = [unicode(cell, 'utf-8') for cell in rowtemp]
            code = int(row[0])
            if code == 50:
                header['makam'] = {'mu2_name':row[7]}
                header['key_signature'] = row[8].split('/')
            elif code == 51:
                header['usul'] = {'mu2_name':row[7], 'mertebe':int(row[3]),
                    'number_of_pulses':int(row[2])}
            elif code == 52:
                try:
                    header['tempo'] = {'value':int(row[4]), 'unit':'bpm'}
                except ValueError:  # the bpm might be a float for low tempo
                    header['tempo'] = {'value':float(row[4]), 'unit':'bpm'}
                if not int(row[3]) == header['usul']['mertebe']:
                    print symbtrname + ': Mertebe and tempo unit are different!'
            elif code == 56:
                header['usul']['subdivision'] = {'mertebe':int(row[3]),
                    'number_of_pulses':int(row[2])}
            elif code == 57:
                header['form'] = {'mu2_name':row[7]}
            elif code == 58:
                header['composer'] = {'mu2_name':row[7]}
            elif code == 59:
                header['lyricist'] = {'mu2_name':row[7]}
            elif code == 60:
                header['title'] = {'mu2_title':row[7]}
            elif code == 62:
                header['genre'] = 'folk' if row[7] == 'E' else 'classical'
            elif code == 63:
                header['notation'] = row[7]
            elif code in range(50, 64):
                print '   Unparsed code: ' + ' '.join(row)
            else:  # end of header
                break

    # get the metadata
    slugs = getSlug(symbtrname)
    header['makam']['symbtr_slug'] = slugs['makam']
    header['form']['symbtr_slug'] = slugs['form']
    header['usul']['symbtr_slug'] = slugs['usul']
    header['title']['symbtr_slug'] = slugs['name']
    header['composer']['symbtr_slug'] = slugs['composer']

    # validate the header content
    makam = getMakam(header['makam']['symbtr_slug'])
    isMakamValid = validateAttribute(header['makam'], makam, symbtrname)

    form = getForm(header['form']['symbtr_slug'])
    isFormValid = validateAttribute(header['form'], form, symbtrname)

    usul = getUsul(header['usul']['symbtr_slug'])
    isUsulValid = validateAttribute(header['usul'], usul, symbtrname)

    isHeaderValid = isMakamValid and isFormValid and isUsulValid

    return header, headerRow, isHeaderValid
<|endoftext|>


<commit_before># Copyright (C) 2019  The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information

from base64 import b64encode
from typing import cast, Optional

from django.conf.urls import url
from django.contrib.staticfiles import finders
from django.http import HttpResponse, HttpRequest

from pybadges import badge

from swh.model.exceptions import ValidationError
from swh.model.identifiers import (
    persistent_identifier, parse_persistent_identifier,
    CONTENT, DIRECTORY, ORIGIN, RELEASE, REVISION, SNAPSHOT
)
from swh.web.common import service
from swh.web.common.exc import BadInputExc, NotFoundExc
from swh.web.common.utils import reverse, resolve_swh_persistent_id

_orange = '#f36a24'
_yellow = '#fac11f'
_red = '#cd5741'

_swh_logo_data = None

_badge_config = {
    CONTENT: {
        'color': _yellow,
        'title': 'Archived source file',
    },
    DIRECTORY: {
        'color': _yellow,
        'title': 'Archived source tree',
    },
    ORIGIN: {
        'color': _orange,
        'title': 'Archived software repository',
    },
    RELEASE: {
        'color': _yellow,
        'title': 'Archived software release',
    },
    REVISION: {
        'color': _yellow,
        'title': 'Archived commit',
    },
    SNAPSHOT: {
        'color': _yellow,
        'title': 'Archived software repository snapshot',
    },
    'error': {
        'color': _red,
        'title': 'An error occurred when generating the badge'
    }
}

def _get_logo_data() -> str:
    """
    Get data-URI for Software Heritage SVG logo to embed it in
    the generated badges.
    """
    global _swh_logo_data
    if _swh_logo_data is None:
        swh_logo_path = cast(str, finders.find('img/swh-logo-white.svg'))
        with open(swh_logo_path, 'rb') as swh_logo_file:
            _swh_logo_data = ('data:image/svg+xml;base64,%s' %
                              b64encode(swh_logo_file.read()).decode('ascii'))
    return _swh_logo_data

def _swh_badge(request: HttpRequest, object_type: str, object_id: str,
               object_pid: Optional[str] = '') -> HttpResponse:
    """
    Generate a Software Heritage badge for a given object type and id.

    Args:
        request: input http request
        object_type: The type of swh object to generate a badge for,
            either *content*, *directory*, *revision*, *release*, *origin*
            or *snapshot*
        object_id: The id of the swh object, either an url for origin
            type or a *sha1* for other object types
        object_pid: If provided, the object persistent
            identifier will not be recomputed

    Returns:
        HTTP response with content type *image/svg+xml* containing the SVG
        badge data. If the provided parameters are invalid, HTTP 400 status
        code will be returned. If the object can not be found in the archive,
        HTTP 404 status code will be returned.

    """
    left_text = 'error'
    whole_link = ''
    status = 200

    try:
        if object_type == ORIGIN:
            service.lookup_origin({'url': object_id})
            right_text = 'repository'
            whole_link = reverse('browse-origin',
                                 url_args={'origin_url': object_id})
        else:
            # when pid is provided, object type and id will be parsed
            # from it
            if object_pid:
                parsed_pid = parse_persistent_identifier(object_pid)
                object_type = parsed_pid.object_type
                object_id = parsed_pid.object_id
            swh_object = service.lookup_object(object_type, object_id)
            if object_pid:
                right_text = object_pid
            else:
                right_text = persistent_identifier(object_type, object_id)

            whole_link = resolve_swh_persistent_id(right_text)['browse_url']
            # remove pid metadata if any for badge text
            if object_pid:
                right_text = right_text.split(';')[0]
            # use release name for badge text
            if object_type == RELEASE:
                right_text = 'release %s' % swh_object['name']
        left_text = 'archived'
    except (BadInputExc, ValidationError):
        right_text = f'invalid {object_type if object_type else "object"} id'
        status = 400
        object_type = 'error'
    except NotFoundExc:
        right_text = f'{object_type if object_type else "object"} not found'
        status = 404
        object_type = 'error'

    badge_data = badge(left_text=left_text,
                       right_text=right_text,
                       right_color=_badge_config[object_type]['color'],
                       whole_link=request.build_absolute_uri(whole_link),
                       whole_title=_badge_config[object_type]['title'],
                       logo=_get_logo_data(),
                       embed_logo=True)

    return HttpResponse(badge_data, content_type='image/svg+xml',
                        status=status)

def _swh_badge_pid(request: HttpRequest, object_pid: str) -> HttpResponse:
    """
    Generate a Software Heritage badge for a given object persistent
    identifier.

    Args:
        request (django.http.HttpRequest): input http request
        object_pid (str): A swh object persistent identifier

    Returns:
        django.http.HttpResponse: An http response with content type
            *image/svg+xml* containing the SVG badge data. If any error
            occurs, a status code of 400 will be returned.
    """
    return _swh_badge(request, '', '', object_pid)

urlpatterns = [
    url(r'^badge/(?P<object_type>[a-z]+)/(?P<object_id>.+)/$', _swh_badge,
        name='swh-badge'),
    url(r'^badge/(?P<object_pid>swh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$',
        _swh_badge_pid, name='swh-badge-pid'),
]
<commit_msg>misc/badges: Use blue color for non origin badges<commit_after># Copyright (C) 2019  The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information

from base64 import b64encode
from typing import cast, Optional

from django.conf.urls import url
from django.contrib.staticfiles import finders
from django.http import HttpResponse, HttpRequest

from pybadges import badge

from swh.model.exceptions import ValidationError
from swh.model.identifiers import (
    persistent_identifier, parse_persistent_identifier,
    CONTENT, DIRECTORY, ORIGIN, RELEASE, REVISION, SNAPSHOT
)
from swh.web.common import service
from swh.web.common.exc import BadInputExc, NotFoundExc
from swh.web.common.utils import reverse, resolve_swh_persistent_id

_orange = '#f36a24'
_blue = '#0172b2'
_red = '#cd5741'

_swh_logo_data = None

_badge_config = {
    CONTENT: {
        'color': _blue,
        'title': 'Archived source file',
    },
    DIRECTORY: {
        'color': _blue,
        'title': 'Archived source tree',
    },
    ORIGIN: {
        'color': _orange,
        'title': 'Archived software repository',
    },
    RELEASE: {
        'color': _blue,
        'title': 'Archived software release',
    },
    REVISION: {
        'color': _blue,
        'title': 'Archived commit',
    },
    SNAPSHOT: {
        'color': _blue,
        'title': 'Archived software repository snapshot',
    },
    'error': {
        'color': _red,
        'title': 'An error occurred when generating the badge'
    }
}

def _get_logo_data() -> str:
    """
    Get data-URI for Software Heritage SVG logo to embed it in
    the generated badges.
    """
    global _swh_logo_data
    if _swh_logo_data is None:
        swh_logo_path = cast(str, finders.find('img/swh-logo-white.svg'))
        with open(swh_logo_path, 'rb') as swh_logo_file:
            _swh_logo_data = ('data:image/svg+xml;base64,%s' %
                              b64encode(swh_logo_file.read()).decode('ascii'))
    return _swh_logo_data

def _swh_badge(request: HttpRequest, object_type: str, object_id: str,
               object_pid: Optional[str] = '') -> HttpResponse:
    """
    Generate a Software Heritage badge for a given object type and id.

    Args:
        request: input http request
        object_type: The type of swh object to generate a badge for,
            either *content*, *directory*, *revision*, *release*, *origin*
            or *snapshot*
        object_id: The id of the swh object, either an url for origin
            type or a *sha1* for other object types
        object_pid: If provided, the object persistent
            identifier will not be recomputed

    Returns:
        HTTP response with content type *image/svg+xml* containing the SVG
        badge data. If the provided parameters are invalid, HTTP 400 status
        code will be returned. If the object can not be found in the archive,
        HTTP 404 status code will be returned.

    """
    left_text = 'error'
    whole_link = ''
    status = 200

    try:
        if object_type == ORIGIN:
            service.lookup_origin({'url': object_id})
            right_text = 'repository'
            whole_link = reverse('browse-origin',
                                 url_args={'origin_url': object_id})
        else:
            # when pid is provided, object type and id will be parsed
            # from it
            if object_pid:
                parsed_pid = parse_persistent_identifier(object_pid)
                object_type = parsed_pid.object_type
                object_id = parsed_pid.object_id
            swh_object = service.lookup_object(object_type, object_id)
            if object_pid:
                right_text = object_pid
            else:
                right_text = persistent_identifier(object_type, object_id)

            whole_link = resolve_swh_persistent_id(right_text)['browse_url']
            # remove pid metadata if any for badge text
            if object_pid:
                right_text = right_text.split(';')[0]
            # use release name for badge text
            if object_type == RELEASE:
                right_text = 'release %s' % swh_object['name']
        left_text = 'archived'
    except (BadInputExc, ValidationError):
        right_text = f'invalid {object_type if object_type else "object"} id'
        status = 400
        object_type = 'error'
    except NotFoundExc:
        right_text = f'{object_type if object_type else "object"} not found'
        status = 404
        object_type = 'error'

    badge_data = badge(left_text=left_text,
                       right_text=right_text,
                       right_color=_badge_config[object_type]['color'],
                       whole_link=request.build_absolute_uri(whole_link),
                       whole_title=_badge_config[object_type]['title'],
                       logo=_get_logo_data(),
                       embed_logo=True)

    return HttpResponse(badge_data, content_type='image/svg+xml',
                        status=status)

def _swh_badge_pid(request: HttpRequest, object_pid: str) -> HttpResponse:
    """
    Generate a Software Heritage badge for a given object persistent
    identifier.

    Args:
        request (django.http.HttpRequest): input http request
        object_pid (str): A swh object persistent identifier

    Returns:
        django.http.HttpResponse: An http response with content type
            *image/svg+xml* containing the SVG badge data. If any error
            occurs, a status code of 400 will be returned.
    """
    return _swh_badge(request, '', '', object_pid)

urlpatterns = [
    url(r'^badge/(?P<object_type>[a-z]+)/(?P<object_id>.+)/$', _swh_badge,
        name='swh-badge'),
    url(r'^badge/(?P<object_pid>swh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$',
        _swh_badge_pid, name='swh-badge-pid'),
]
<|endoftext|>