<commit_before>import json
import requests
from django.core.mail import send_mail
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.loader import get_template
from django.template import Context
from celery import shared_task
from celery.task import Task
from celery.signals import task_failure
from myhpom.models.document import DocumentUrl
from myhpom.models import CloudFactoryDocumentRun

@shared_task
class CloudFactorySubmitDocumentRun(Task):
    def run(self, document_url_id, document_host=None):
        """
        ## Arguments
        * document_url_id = the database id for the DocumentUrl that is being submitted
        * document_host = the host (incl. protocol/scheme) to use for the DocumentURL & callback
            -- configurable so that we can tell CloudFactory which server to interact with
            (the document_host and the callback_host are assumed the same)

        ## Task Process
        - submit the DocumentUrl to the CloudFactory (test) endpoint
        - store the status of the submitted document
        - if the DocumentUrl no longer exists, abort the task without an Exception.
            - the DocumentUrl might no longer exist by the time celery picks it up,
            - or the document it points to might have been removed.
        - if an error occurs in this process, send support email.
        """
        cf_run = CloudFactoryDocumentRun(document_host=document_host or '')

        # If the DocumentUrl has been deleted, abort the task with a message
        # -- no need to admin support email, this is an expected case.
        try:
            cf_run.document_url = DocumentUrl.objects.get(id=document_url_id)
        except DocumentUrl.DoesNotExist:
            cf_run.status = CloudFactoryDocumentRun.STATUS_DELETED
            cf_run.save()
            return cf_run.id

        # Any exception in the rest of the task should result in support email
        cf_run.post_data = cf_run.create_post_data()
        cf_run.save()

        try:
            response = requests.post(settings.CLOUDFACTORY_API_URL + '/runs', json=cf_run.post_data)
        except requests.RequestException:
            cf_run.status = CloudFactoryDocumentRun.STATUS_REQ_ERROR
            cf_run.save()
            raise

        if response.status_code == 422:
            cf_run.status = CloudFactoryDocumentRun.STATUS_UNPROCESSABLE
        elif response.status_code == 404:
            cf_run.status = CloudFactoryDocumentRun.STATUS_NOTFOUND
        elif response.status_code == 201:
            cf_run.status = CloudFactoryDocumentRun.STATUS_PROCESSING
        else:
            cf_run.status = CloudFactoryDocumentRun.STATUS_ERROR

        cf_run.response_content = response.content
        cf_run.save()

        # if the run could not be created (status_code != 201), send support email
        # -- we need to understand why the the run could not be created at CloudFactory.
        if response.status_code != 201:
            raise ValueError(
                "== POST DATA ==\n%s\n\n== RESPONSE CONTENT ==\n%s"
                % (json.dumps(cf_run.post_data, indent=2), cf_run.response_content)
            )

        # following throws an error (as it should) if response.content is not json-parsable.
        cf_run.save_response_data(response.content)

        return cf_run.pk

@shared_task
class CloudFactoryUpdateDocumentRun(Task):
    def run(self, cf_run_id):
        """Update the CloudFactoryDocumentRun object from the CF API.

        ## Use Cases
        * The admin user can update the status of all or selected runs from CloudFactory.

        ## Parameters
        * cf_run_id = the id (not the run_id) of the CloudFactoryDocumentRun object.
        """
        cf_run = CloudFactoryDocumentRun.objects.get(id=cf_run_id)

        if cf_run.status in [
            CloudFactoryDocumentRun.STATUS_PROCESSING,  # still going last we checked
            CloudFactoryDocumentRun.STATUS_REQ_ERROR,  # didn't work last time; was it created?
            CloudFactoryDocumentRun.STATUS_ABORTED,  # we previously aborted; did it take?
        ]:
            try:
                run_url = "%s/runs/%s" % (settings.CLOUDFACTORY_API_URL, cf_run.run_id)
                response = requests.get(run_url)
            except requests.RequestException:
                cf_run.status = CloudFactoryDocumentRun.STATUS_REQ_ERROR
                cf_run.save()
                raise

            if response.status_code == 404:
                cf_run.status = CloudFactoryDocumentRun.STATUS_NOTFOUND
                cf_run.response_content = response.content  # not json, shouldn't raise Exception.
                cf_run.save()
            elif response.status_code == 200:
                cf_run.save_response_data(response.content)
            else:
                cf_run.status = CloudFactoryDocumentRun.STATUS_ERROR
                cf_run.save()
                raise ValueError(
                    """URL: %s\nResponse status: %d\nResponse Data: %s"""
                    % (response.url, response.status_code, response.content)
                )

@shared_task
class CloudFactoryAbortDocumentRun(Task):
    def run(self, cf_run_id):
        """Abort the given CloudFactoryDocumentRun.

        ## Use Cases:
        * when the user deletes their DocumentUrl, this task is triggered automatically
            (through the deletion of the DocumentUrl).
        * can be called from the admin

        ## Parameters:
        * cf_run_id = the id (not the run_id) of a CloudFactoryDocumentRun object

        ## Process:
        * If it is in progress (status='Processing'), cancel it at CloudFactory.
            * 202 = CloudFactory accepted the request
            * 405 = already processed / aborted at CloudFactory
        * Get the run info from CloudFactory and update our models.
            * 200 = Ok
        * Any response status apart from those given should raise an exception / send error email,
            because it means we have a bug / false assumption somewhere in our system.
        """
        cf_run = CloudFactoryDocumentRun.objects.get(id=cf_run_id)
        if cf_run.status in [
            CloudFactoryDocumentRun.STATUS_PROCESSING,  # still going last we checked
            CloudFactoryDocumentRun.STATUS_REQ_ERROR,  # didn't work last time; was it created?
        ]:
            # abort the run
            abort_url = "%s/runs/%s/abort" % (settings.CLOUDFACTORY_API_URL, cf_run.run_id)
            response = requests.post(abort_url)  # yes, CF wants a POST without a body for this.

            if response.status_code == 404:
                cf_run.status = CloudFactoryDocumentRun.STATUS_NOTFOUND
                cf_run.save()  # don't save the response.content, it's probably not json
            elif response.status_code == 202:
                # 202 = they accepted the response, so we can consider it done.
                cf_run.status = CloudFactoryDocumentRun.STATUS_ABORTED
                cf_run.save()
            elif response.status_code == 405:
                # 405 = CF uses it to mean "already done, folks" -- whether aborted or processed
                # (the details of what was done are in the .response_content)
                cf_run.status = CloudFactoryDocumentRun.STATUS_PROCESSED
                cf_run.save()
            else:
                cf_run.status = CloudFactoryDocumentRun.STATUS_ERROR
                cf_run.save()
                raise ValueError(
                    """URL: %s\nResponse status: %d\nResponse Content:\n%s"""
                    % (response.url, response.status_code, response.content)
                )

@shared_task
class EmailUserDocumentReviewCompleted(Task):
    def run(self, cfrun_id, scheme, host):
        """Send the user an email indicating that the review has been completed.
        """
        # if the document has been deleted, don't send a notification.
        # (e.g., The user might have deleted the document while the run was at CF.)
        cfrun = CloudFactoryDocumentRun.objects.get(id=cfrun_id)
        if cfrun.document_url:
            email = cfrun.document_url.advancedirective.user.email
            subject = "Mind My Health: Document Review Completed"
            context = Context(
                {'dashboard_url': "%s://%s%s" % (scheme, host, reverse('myhpom:dashboard'))}
            )
            template = get_template('myhpom/profile/document_review_completed_email.txt')
            message = template.render(context)
            send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [email])

# == SIGNALS ==

@task_failure.connect()
def send_celery_error_mail(task_id=None, einfo=None, sender=None, *args, **kwargs):
    task = sender.name
    subject = '{prefix}[celery] Error: {task} ({task_id})'.format(
        prefix=settings.EMAIL_SUBJECT_PREFIX, task=task, task_id=task_id
    )
    context = Context({'traceback': einfo, 'task': task})
    message = get_template('myhpom/celery_task_error_email.txt').render(context)
    send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [settings.DEFAULT_SUPPORT_EMAIL])
<commit_msg>[MH-288] Fixes race condition with abort task.<commit_after>import json
import requests
from django.core.mail import send_mail
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.loader import get_template
from django.template import Context
from celery import shared_task
from celery.task import Task
from celery.signals import task_failure
from myhpom.models.document import DocumentUrl
from myhpom.models import CloudFactoryDocumentRun

@shared_task
class CloudFactorySubmitDocumentRun(Task):
    def run(self, document_url_id, document_host=None):
        """
        ## Arguments
        * document_url_id = the database id for the DocumentUrl that is being submitted
        * document_host = the host (incl. protocol/scheme) to use for the DocumentURL & callback
            -- configurable so that we can tell CloudFactory which server to interact with
            (the document_host and the callback_host are assumed the same)

        ## Task Process
        - submit the DocumentUrl to the CloudFactory (test) endpoint
        - store the status of the submitted document
        - if the DocumentUrl no longer exists, abort the task without an Exception.
            - the DocumentUrl might no longer exist by the time celery picks it up,
            - or the document it points to might have been removed.
        - if an error occurs in this process, send support email.
        """
        cf_run = CloudFactoryDocumentRun(document_host=document_host or '')

        # If the DocumentUrl has been deleted, abort the task with a message
        # -- no need to admin support email, this is an expected case.
        try:
            cf_run.document_url = DocumentUrl.objects.get(id=document_url_id)
        except DocumentUrl.DoesNotExist:
            cf_run.status = CloudFactoryDocumentRun.STATUS_DELETED
            cf_run.save()
            return cf_run.id

        # Any exception in the rest of the task should result in support email
        cf_run.post_data = cf_run.create_post_data()
        cf_run.save()

        try:
            response = requests.post(settings.CLOUDFACTORY_API_URL + '/runs', json=cf_run.post_data)
        except requests.RequestException:
            cf_run.status = CloudFactoryDocumentRun.STATUS_REQ_ERROR
            cf_run.save()
            raise

        if response.status_code == 422:
            cf_run.status = CloudFactoryDocumentRun.STATUS_UNPROCESSABLE
        elif response.status_code == 404:
            cf_run.status = CloudFactoryDocumentRun.STATUS_NOTFOUND
        elif response.status_code == 201:
            cf_run.status = CloudFactoryDocumentRun.STATUS_PROCESSING
        else:
            cf_run.status = CloudFactoryDocumentRun.STATUS_ERROR

        cf_run.response_content = response.content
        cf_run.save()

        # if the run could not be created (status_code != 201), send support email
        # -- we need to understand why the the run could not be created at CloudFactory.
        if response.status_code != 201:
            raise ValueError(
                "== POST DATA ==\n%s\n\n== RESPONSE CONTENT ==\n%s"
                % (json.dumps(cf_run.post_data, indent=2), cf_run.response_content)
            )

        # following throws an error (as it should) if response.content is not json-parsable.
        cf_run.save_response_data(response.content)

        return cf_run.pk

@shared_task
class CloudFactoryUpdateDocumentRun(Task):
    def run(self, cf_run_id):
        """Update the CloudFactoryDocumentRun object from the CF API.

        ## Use Cases
        * The admin user can update the status of all or selected runs from CloudFactory.

        ## Parameters
        * cf_run_id = the id (not the run_id) of the CloudFactoryDocumentRun object.
        """
        cf_run = CloudFactoryDocumentRun.objects.get(id=cf_run_id)

        if cf_run.status in [
            CloudFactoryDocumentRun.STATUS_PROCESSING,  # still going last we checked
            CloudFactoryDocumentRun.STATUS_REQ_ERROR,  # didn't work last time; was it created?
            CloudFactoryDocumentRun.STATUS_ABORTED,  # we previously aborted; did it take?
        ]:
            try:
                run_url = "%s/runs/%s" % (settings.CLOUDFACTORY_API_URL, cf_run.run_id)
                response = requests.get(run_url)
            except requests.RequestException:
                cf_run.status = CloudFactoryDocumentRun.STATUS_REQ_ERROR
                cf_run.save()
                raise

            if response.status_code == 404:
                cf_run.status = CloudFactoryDocumentRun.STATUS_NOTFOUND
                cf_run.response_content = response.content  # not json, shouldn't raise Exception.
                cf_run.save()
            elif response.status_code == 200:
                cf_run.save_response_data(response.content)
            else:
                cf_run.status = CloudFactoryDocumentRun.STATUS_ERROR
                cf_run.save()
                raise ValueError(
                    """URL: %s\nResponse status: %d\nResponse Data: %s"""
                    % (response.url, response.status_code, response.content)
                )

@shared_task
class CloudFactoryAbortDocumentRun(Task):
    def run(self, cf_run_id):
        """Abort the given CloudFactoryDocumentRun.

        ## Use Cases:
        * when the user deletes their DocumentUrl, this task is triggered automatically
            (through the deletion of the DocumentUrl).
        * can be called from the admin

        ## Parameters:
        * cf_run_id = the id (not the run_id) of a CloudFactoryDocumentRun object

        ## Process:
        * If it is in progress (status='Processing'), cancel it at CloudFactory.
            * 202 = CloudFactory accepted the request
            * 405 = already processed / aborted at CloudFactory
        * Get the run info from CloudFactory and update our models.
            * 200 = Ok
        * Any response status apart from those given should raise an exception / send error email,
            because it means we have a bug / false assumption somewhere in our system.
        """
        cf_run = CloudFactoryDocumentRun.objects.get(id=cf_run_id)
        if cf_run.status in [
            CloudFactoryDocumentRun.STATUS_PROCESSING,  # still going last we checked
            CloudFactoryDocumentRun.STATUS_REQ_ERROR,  # didn't work last time; was it created?
        ]:
            abort_url = "%s/runs/%s/abort" % (settings.CLOUDFACTORY_API_URL, cf_run.run_id)
            response = requests.post(abort_url)  # yes, CF wants a POST without a body for this.

            if response.status_code == 404:
                CloudFactoryDocumentRun.objects.filter(pk=cf_run_id).update(status=CloudFactoryDocumentRun.STATUS_NOTFOUND)
            elif response.status_code == 202:
                # 202 = they accepted the response, so we can consider it done.
                CloudFactoryDocumentRun.objects.filter(pk=cf_run_id).update(status=CloudFactoryDocumentRun.STATUS_ABORTED)
            elif response.status_code == 405:
                # 405 = CF uses it to mean "already done, folks" -- whether aborted or processed
                CloudFactoryDocumentRun.objects.filter(pk=cf_run_id).update(status=CloudFactoryDocumentRun.STATUS_PROCESSED)
            else:
                CloudFactoryDocumentRun.objects.filter(pk=cf_run_id).update(status=CloudFactoryDocumentRun.STATUS_ERROR)
                raise ValueError(
                    """URL: %s\nResponse status: %d\nResponse Content:\n%s"""
                    % (response.url, response.status_code, response.content)
                )

@shared_task
class EmailUserDocumentReviewCompleted(Task):
    def run(self, cfrun_id, scheme, host):
        """Send the user an email indicating that the review has been completed.
        """
        # if the document has been deleted, don't send a notification.
        # (e.g., The user might have deleted the document while the run was at CF.)
        cfrun = CloudFactoryDocumentRun.objects.get(id=cfrun_id)
        if cfrun.document_url:
            email = cfrun.document_url.advancedirective.user.email
            subject = "Mind My Health: Document Review Completed"
            context = Context(
                {'dashboard_url': "%s://%s%s" % (scheme, host, reverse('myhpom:dashboard'))}
            )
            template = get_template('myhpom/profile/document_review_completed_email.txt')
            message = template.render(context)
            send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [email])

# == SIGNALS ==

@task_failure.connect()
def send_celery_error_mail(task_id=None, einfo=None, sender=None, *args, **kwargs):
    task = sender.name
    subject = '{prefix}[celery] Error: {task} ({task_id})'.format(
        prefix=settings.EMAIL_SUBJECT_PREFIX, task=task, task_id=task_id
    )
    context = Context({'traceback': einfo, 'task': task})
    message = get_template('myhpom/celery_task_error_email.txt').render(context)
    send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [settings.DEFAULT_SUPPORT_EMAIL])
<|endoftext|>


<commit_before>import warnings
from . import base

class TestType(base.SchemaTestCase):

    def test_single_type(self):
        schema = {'type': 'string'}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_single_type_unicode(self):
        schema = {u'type': u'string'}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_no_type(self):
        schema = {'title': 'ambiguous schema'}
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            self.add_schema(schema)
        self.assertResult(schema)

class TestAnyOf(base.SchemaTestCase):

    def test_multi_type(self):
        schema = {'type': ['boolean', 'null', 'number', 'string']}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_anyof(self):
        schema = {"anyOf": [
            {"type": "null"},
            {"type": "boolean", "title": "Gruyere"}
        ]}
        self.add_schema(schema)
        self.assertResult(schema)

class TestPreserveKeys(base.SchemaTestCase):

    def test_preserves_existing_keys(self):
        schema = {'type': 'number', 'const': 5, 'my-custom-key': True}
        self.add_schema(schema)
        self.assertResult(schema)
<commit_msg>add tests for extra keyword preservation<commit_after>import warnings
from . import base

class TestType(base.SchemaTestCase):

    def test_single_type(self):
        schema = {'type': 'string'}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_single_type_unicode(self):
        schema = {u'type': u'string'}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_no_type(self):
        schema = {'title': 'ambiguous schema'}
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            self.add_schema(schema)
        self.assertResult(schema)

class TestAnyOf(base.SchemaTestCase):

    def test_multi_type(self):
        schema = {'type': ['boolean', 'null', 'number', 'string']}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_multi_type_with_extra_keywords(self):
        schema = {'type': ['boolean', 'null', 'number', 'string'],
                  'title': 'this will be duplicated'}
        self.add_schema(schema)
        self.assertResult({'anyOf': [
            {'type': 'boolean', 'title': 'this will be duplicated'},
            {'type': 'null', 'title': 'this will be duplicated'},
            {'type': 'number', 'title': 'this will be duplicated'},
            {'type': 'string', 'title': 'this will be duplicated'}
        ]})

    def test_anyof(self):
        schema = {"anyOf": [
            {"type": "null"},
            {"type": "boolean", "title": "Gruyere"}
        ]}
        self.add_schema(schema)
        self.assertResult(schema)

class TestPreserveExtraKeywords(base.SchemaTestCase):

    def test_basic_type(self):
        schema = {'type': 'boolean', 'const': False, 'myKeyword': True}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_number(self):
        schema = {'type': 'number', 'const': 5, 'myKeyword': True}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_list(self):
        schema = {'type': 'array', 'items': {},
                  'const': 5, 'myKeyword': True}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_tuple(self):
        schema = {'type': 'array', 'items': [],
                  'const': 5, 'myKeyword': True}
        self.add_schema(schema)
        self.assertResult(schema)

    def test_object(self):
        schema = {'type': 'object', 'properties': {},
                  'const': 5, 'myKeyword': True}
        self.add_schema(schema)
        self.assertResult(schema)
<|endoftext|>


<commit_before># The MIT License (MIT)
#
# Copyright (c) 2016 PyBER
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import sys
import libs.reader
from PyQt4 import QtGui
from PyQt4 import QtCore
from pyqtgraph.dockarea import *

class AdvTreeView(QtGui.QTreeView):
    wBER      = []
    wFER      = []
    wBEFE     = []
    wThr      = []
    wLege     = []
    fsWatcher = []

    dataSNR  = []
    dataBER  = []
    dataFER  = []
    dataBEFE = []
    dataThr  = []
    dataLege = []

    #               1  2  3  4  5  6  7  8  9  10  11  12  13  14  15, 16
    colors       = [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17]
    lastSNR      = []
    paths        = []
    styles       = [QtCore.Qt.SolidLine, QtCore.Qt.DashLine, QtCore.Qt.DotLine, QtCore.Qt.DashDotLine, QtCore.Qt.DashDotDotLine]
    dashPatterns = [[1, 3, 4, 3], [2, 3, 4, 3], [1, 3, 1, 3], [4, 3, 4, 3], [3, 3, 2, 3], [4, 3, 1, 3]]

    def getPathId(self, path):
        if path in self.paths:
            curId = 0
            for p in self.paths:
                if p == path:
                    return curId
                else:
                    curId = curId +1
            return -1
        else:
            return -1

    def updateData(self, path):
        if path in self.paths:
            pathId = self.getPathId(path)

            self.dataSNR[pathId], self.dataBER[pathId], self.dataFER[pathId], self.dataBEFE[pathId], self.dataThr[pathId], self.dataLege[pathId] = libs.reader.dataReader(path)

    def plotCurve(self, pathId, dataSNR, dataBER, dataFER, dataBEFE, dataThr):
        icolor = self.colors[pathId % len(self.colors)]
        pen = pg.mkPen(color=(icolor,8), width=2, style=QtCore.Qt.CustomDashLine)
        pen.setDashPattern(self.dashPatterns[pathId % len(self.dashPatterns)])

        self.wBER. plot(x=dataSNR, y=dataBER,  pen=pen, symbol='x', name='BER plot'  )
        self.wFER. plot(x=dataSNR, y=dataFER,  pen=pen, symbol='x', name='FER plot'  )
        self.wBEFE.plot(x=dataSNR, y=dataBEFE, pen=pen, symbol='x', name='BE/FE plot')
        self.wThr. plot(x=dataSNR, y=dataThr,  pen=pen, symbol='x', name='T/P plot'  )

    def updateCurve(self, path):
        if path in self.paths:
            pathId = self.getPathId(path)

            if self.dataSNR[pathId]:
                if self.dataSNR[pathId][len(self.dataSNR[pathId]) -1] > self.lastSNR[pathId]:
                    dataSNR  = self.dataSNR [pathId]
                    dataBER  = self.dataBER [pathId]
                    dataFER  = self.dataFER [pathId]
                    dataThr  = self.dataThr [pathId]
                    dataBEFE = self.dataBEFE[pathId]

                    nPop = 0
                    for i in range(len(dataSNR)):
                        if self.lastSNR[pathId] >= dataSNR[i]:
                            nPop = i

                    for i in range(nPop):
                        dataSNR .pop(0)
                        dataBER .pop(0)
                        dataFER .pop(0)
                        dataBEFE.pop(0)
                        dataThr .pop(0)

                    self.plotCurve(pathId, dataSNR, dataBER, dataFER, dataBEFE, dataThr)
                    self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]
                elif self.dataSNR[pathId][len(self.dataSNR[pathId]) -1] < self.lastSNR[pathId]:
                    self.updateCurves()
                    self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]

    def updateCurves(self):
        self.wBER .clearPlots()
        self.wFER .clearPlots()
        self.wBEFE.clearPlots()
        self.wThr .clearPlots()

        for pathId in range(len(self.paths)):
            self.plotCurve(pathId, self.dataSNR[pathId], self.dataBER[pathId], self.dataFER[pathId], self.dataBEFE[pathId], self.dataThr[pathId])

    def updateDataAndCurve(self, path):
        self.updateData(path)
        self.updateCurve(path)

    def updateLegends(self):
        self.wLege.clear()

        for pathId in range(len(self.paths)):
            icolor = self.colors[pathId % len(self.colors)]
            path   = self.paths[pathId]

            # for filename in self.paths:
            pen = pg.mkPen(color=(icolor,8), width=2, style=QtCore.Qt.CustomDashLine)
            pen.setDashPattern(self.dashPatterns[pathId % len(self.dashPatterns)])

            legendArea = DockArea()
            dCurve     = Dock("", size=(250,600))
            dInfo      = Dock("", size=(250,900))

            legendArea.addDock(dCurve, 'top'   )
            legendArea.addDock(dInfo,  'bottom')

            layoutLegend = QtGui.QFormLayout()
            for entry in self.dataLege[pathId]:
                if len(entry) == 2 and entry[1]:
                    if entry[0] == "Run command":
                        runCmd = QtGui.QLineEdit(str(entry[1]))
                        runCmd.setReadOnly(True)
                        layoutLegend.addRow("" + entry[0] + ": ", runCmd)
                    else:
                        layoutLegend.addRow("" + entry[0] + ": ", QtGui.QLabel(entry[1]))
            wCur = QtGui.QWidget();
            wCur.setLayout(layoutLegend)

            sCur = QtGui.QScrollArea()
            sCur.setWidget(wCur)
            sCur.setWidgetResizable(True)
            dInfo.addWidget(sCur)

            wLegend = pg.PlotWidget(title="BER preview")
            wLegend.setLogMode(False, True)
            wLegend.plot(x=self.dataSNR[pathId], y=self.dataBER[pathId], pen=pen, name='BER plot')
            wLegend.showGrid(False, False)
            dCurve.addWidget(wLegend)

            self.wLege.addTab(legendArea, 'Curve ' + str(pathId +1))

    def selectionChanged(self, selected, deselected):
        super().selectionChanged(selected, deselected)
        newPaths = [ self.model().filePath(index) for index in self.selectedIndexes()
                        if not self.model().isDir(index)] # TODO: remove this restriction

        self.dataSNR  = [0 for x in range(len(newPaths))]
        self.dataBER  = [0 for x in range(len(newPaths))]
        self.dataFER  = [0 for x in range(len(newPaths))]
        self.dataBEFE = [0 for x in range(len(newPaths))]
        self.dataThr  = [0 for x in range(len(newPaths))]
        self.dataLege = [0 for x in range(len(newPaths))]
        self.lastSNR  = [0 for x in range(len(newPaths))]

        pathsToRemove = []
        for p in self.paths:
            if p not in newPaths:
                pathsToRemove.append(p)

        for p in pathsToRemove:
            pId = self.getPathId(p)
            self.paths.pop(pId)

        pathsToAdd = []
        for p in newPaths:
            if p not in self.paths:
                pathsToAdd.append(p)

        for p in pathsToAdd:
            self.paths.append(p)

        if len(pathsToRemove) > 0:
            self.fsWatcher.removePaths(pathsToRemove)
        if len(pathsToAdd) > 0:
            self.fsWatcher.addPaths(pathsToAdd)
        self.fsWatcher.fileChanged.connect(self.updateDataAndCurve)

        for path in self.paths:
            self.updateData(path)

        for pathId in range(len(self.paths)):
            self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]

        self.updateCurves()
        self.updateLegends()

def generatePannel():
    if len(sys.argv) >= 2:
        os.chdir(sys.argv[1])
    else:
        os.chdir("data/")

    model = QtGui.QFileSystemModel()
    model.setReadOnly(True)
    model.setRootPath(QtCore.QDir.currentPath())
    model.setNameFilters(['*.perf', '*.dat', '*.txt', '*.data'])
    model.setNameFilterDisables(False)

    view = AdvTreeView()
    view.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
    view.setModel(model)
    view.hideColumn(1);
    view.hideColumn(2);
    view.hideColumn(3);
    view.setColumnWidth(30, 1)
    view.setRootIndex(model.index(QtCore.QDir.currentPath(), 0))
    view.setAnimated(True)
    view.setIconSize(QtCore.QSize(24,24))

    return view<commit_msg>Import pyqtgraph.<commit_after># The MIT License (MIT)
#
# Copyright (c) 2016 PyBER
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import sys
import libs.reader
import pyqtgraph as pg
from PyQt4 import QtGui
from PyQt4 import QtCore
from pyqtgraph.dockarea import *

class AdvTreeView(QtGui.QTreeView):
    wBER      = []
    wFER      = []
    wBEFE     = []
    wThr      = []
    wLege     = []
    fsWatcher = []

    dataSNR  = []
    dataBER  = []
    dataFER  = []
    dataBEFE = []
    dataThr  = []
    dataLege = []

    #               1  2  3  4  5  6  7  8  9  10  11  12  13  14  15, 16
    colors       = [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17]
    lastSNR      = []
    paths        = []
    styles       = [QtCore.Qt.SolidLine, QtCore.Qt.DashLine, QtCore.Qt.DotLine, QtCore.Qt.DashDotLine, QtCore.Qt.DashDotDotLine]
    dashPatterns = [[1, 3, 4, 3], [2, 3, 4, 3], [1, 3, 1, 3], [4, 3, 4, 3], [3, 3, 2, 3], [4, 3, 1, 3]]

    def getPathId(self, path):
        if path in self.paths:
            curId = 0
            for p in self.paths:
                if p == path:
                    return curId
                else:
                    curId = curId +1
            return -1
        else:
            return -1

    def updateData(self, path):
        if path in self.paths:
            pathId = self.getPathId(path)

            self.dataSNR[pathId], self.dataBER[pathId], self.dataFER[pathId], self.dataBEFE[pathId], self.dataThr[pathId], self.dataLege[pathId] = libs.reader.dataReader(path)

    def plotCurve(self, pathId, dataSNR, dataBER, dataFER, dataBEFE, dataThr):
        icolor = self.colors[pathId % len(self.colors)]
        pen = pg.mkPen(color=(icolor,8), width=2, style=QtCore.Qt.CustomDashLine)
        pen.setDashPattern(self.dashPatterns[pathId % len(self.dashPatterns)])

        self.wBER. plot(x=dataSNR, y=dataBER,  pen=pen, symbol='x', name='BER plot'  )
        self.wFER. plot(x=dataSNR, y=dataFER,  pen=pen, symbol='x', name='FER plot'  )
        self.wBEFE.plot(x=dataSNR, y=dataBEFE, pen=pen, symbol='x', name='BE/FE plot')
        self.wThr. plot(x=dataSNR, y=dataThr,  pen=pen, symbol='x', name='T/P plot'  )

    def updateCurve(self, path):
        if path in self.paths:
            pathId = self.getPathId(path)

            if self.dataSNR[pathId]:
                if self.dataSNR[pathId][len(self.dataSNR[pathId]) -1] > self.lastSNR[pathId]:
                    dataSNR  = self.dataSNR [pathId]
                    dataBER  = self.dataBER [pathId]
                    dataFER  = self.dataFER [pathId]
                    dataThr  = self.dataThr [pathId]
                    dataBEFE = self.dataBEFE[pathId]

                    nPop = 0
                    for i in range(len(dataSNR)):
                        if self.lastSNR[pathId] >= dataSNR[i]:
                            nPop = i

                    for i in range(nPop):
                        dataSNR .pop(0)
                        dataBER .pop(0)
                        dataFER .pop(0)
                        dataBEFE.pop(0)
                        dataThr .pop(0)

                    self.plotCurve(pathId, dataSNR, dataBER, dataFER, dataBEFE, dataThr)
                    self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]
                elif self.dataSNR[pathId][len(self.dataSNR[pathId]) -1] < self.lastSNR[pathId]:
                    self.updateCurves()
                    self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]

    def updateCurves(self):
        self.wBER .clearPlots()
        self.wFER .clearPlots()
        self.wBEFE.clearPlots()
        self.wThr .clearPlots()

        for pathId in range(len(self.paths)):
            self.plotCurve(pathId, self.dataSNR[pathId], self.dataBER[pathId], self.dataFER[pathId], self.dataBEFE[pathId], self.dataThr[pathId])

    def updateDataAndCurve(self, path):
        self.updateData(path)
        self.updateCurve(path)

    def updateLegends(self):
        self.wLege.clear()

        for pathId in range(len(self.paths)):
            icolor = self.colors[pathId % len(self.colors)]
            path   = self.paths[pathId]

            # for filename in self.paths:
            pen = pg.mkPen(color=(icolor,8), width=2, style=QtCore.Qt.CustomDashLine)
            pen.setDashPattern(self.dashPatterns[pathId % len(self.dashPatterns)])

            legendArea = DockArea()
            dCurve     = Dock("", size=(250,600))
            dInfo      = Dock("", size=(250,900))

            legendArea.addDock(dCurve, 'top'   )
            legendArea.addDock(dInfo,  'bottom')

            layoutLegend = QtGui.QFormLayout()
            for entry in self.dataLege[pathId]:
                if len(entry) == 2 and entry[1]:
                    if entry[0] == "Run command":
                        runCmd = QtGui.QLineEdit(str(entry[1]))
                        runCmd.setReadOnly(True)
                        layoutLegend.addRow("" + entry[0] + ": ", runCmd)
                    else:
                        layoutLegend.addRow("" + entry[0] + ": ", QtGui.QLabel(entry[1]))
            wCur = QtGui.QWidget();
            wCur.setLayout(layoutLegend)

            sCur = QtGui.QScrollArea()
            sCur.setWidget(wCur)
            sCur.setWidgetResizable(True)
            dInfo.addWidget(sCur)

            wLegend = pg.PlotWidget(title="BER preview")
            wLegend.setLogMode(False, True)
            wLegend.plot(x=self.dataSNR[pathId], y=self.dataBER[pathId], pen=pen, name='BER plot')
            wLegend.showGrid(False, False)
            dCurve.addWidget(wLegend)

            self.wLege.addTab(legendArea, 'Curve ' + str(pathId +1))

    def selectionChanged(self, selected, deselected):
        super().selectionChanged(selected, deselected)
        newPaths = [ self.model().filePath(index) for index in self.selectedIndexes()
                        if not self.model().isDir(index)] # TODO: remove this restriction

        self.dataSNR  = [0 for x in range(len(newPaths))]
        self.dataBER  = [0 for x in range(len(newPaths))]
        self.dataFER  = [0 for x in range(len(newPaths))]
        self.dataBEFE = [0 for x in range(len(newPaths))]
        self.dataThr  = [0 for x in range(len(newPaths))]
        self.dataLege = [0 for x in range(len(newPaths))]
        self.lastSNR  = [0 for x in range(len(newPaths))]

        pathsToRemove = []
        for p in self.paths:
            if p not in newPaths:
                pathsToRemove.append(p)

        for p in pathsToRemove:
            pId = self.getPathId(p)
            self.paths.pop(pId)

        pathsToAdd = []
        for p in newPaths:
            if p not in self.paths:
                pathsToAdd.append(p)

        for p in pathsToAdd:
            self.paths.append(p)

        if len(pathsToRemove) > 0:
            self.fsWatcher.removePaths(pathsToRemove)
        if len(pathsToAdd) > 0:
            self.fsWatcher.addPaths(pathsToAdd)
        self.fsWatcher.fileChanged.connect(self.updateDataAndCurve)

        for path in self.paths:
            self.updateData(path)

        for pathId in range(len(self.paths)):
            self.lastSNR[pathId] = self.dataSNR[pathId][len(self.dataSNR[pathId]) -1]

        self.updateCurves()
        self.updateLegends()

def generatePannel():
    if len(sys.argv) >= 2:
        os.chdir(sys.argv[1])
    else:
        os.chdir("data/")

    model = QtGui.QFileSystemModel()
    model.setReadOnly(True)
    model.setRootPath(QtCore.QDir.currentPath())
    model.setNameFilters(['*.perf', '*.dat', '*.txt', '*.data'])
    model.setNameFilterDisables(False)

    view = AdvTreeView()
    view.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
    view.setModel(model)
    view.hideColumn(1);
    view.hideColumn(2);
    view.hideColumn(3);
    view.setColumnWidth(30, 1)
    view.setRootIndex(model.index(QtCore.QDir.currentPath(), 0))
    view.setAnimated(True)
    view.setIconSize(QtCore.QSize(24,24))

    return view<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
## @package ivf.batch.relighting_figure
#
#  ivf.batch.relighting_figure utility package.
#  @author      tody
#  @date        2016/02/22

import numpy as np
import matplotlib.pyplot as plt
import cv2

from ivf.np.norm import normalizeVector
from ivf.plot.window import SubplotGrid, showMaximize
from ivf.datasets.shape import shapeFile, shapeResultFile
from ivf.io_util.image import loadNormal
from ivf.datasets.colormap import colorMapFile, loadColorMap
from ivf.core.shader.toon import ColorMapShader
from ivf.core.sfs.toon_sfs import ToonSFS
from ivf.batch.shape_estimation_figure import lumoSFS, wuSFS, estimatedReflectance
from ivf.core.shader.shader import LdotN
from ivf.cv.image import setAlpha, to32F
from ivf.core.shader.light_sphere import lightSphere
from ivf.plot.fig2np import figure2numpy
from ivf.io_util.video import saveVideo

def relightingVideo(shape_name="Ogre", cmap_id=3):
    num_methods = 3
    num_rows = 1
    num_cols = num_methods + 2

    num_lights = 120

    w = 10
    h = 5

    fig, axes = plt.subplots(figsize=(w, h))
    font_size = 15
    fig.subplots_adjust(left=0.02, right=0.98, top=0.96, bottom=0.04, hspace=0.15, wspace=0.1)
    fig.suptitle("Shading Analysis", fontsize=font_size)

    plot_grid = SubplotGrid(num_rows, num_cols)

    Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
    Lg_img = lightSphere(Lg)

    L1 = normalizeVector(np.array([0.5, 0.5, 0.6]))

    Ls = [normalizeVector(Lg * (1.0 - t) + t * L1) for t in np.linspace(0.0, 1.0, num_lights) ]
    # Ls = [normalizeVector(Lg + 1.0 * np.cos(t) * np.array([1, 0, 0]) + 1.0 * np.sin(t) * np.array([0, 1, 0])) for t in np.linspace(0.0, 1.0, num_lights) ]

    Ng_data = shapeFile(shape_name)
    Ng_data = loadNormal(Ng_data)
    Ng_32F, A_8U = Ng_data

    N0_file = shapeResultFile(result_name="InitialNormal", data_name=shape_name)
    N0_data = loadNormal(N0_file)
    N0_32F, A_8U = N0_data
    A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)

    colormap_file = colorMapFile(cmap_id)
    M_32F = loadColorMap(colormap_file)
    C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)

    toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
    toon_sfs.setInitialNormal(N0_32F)
    toon_sfs.setNumIterations(iterations=50)
    toon_sfs.setWeights(w_lap=0.2)
    toon_sfs.run()

    N_toon = toon_sfs.normal()
    C_toon = toon_sfs.shading()

    C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
    C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)

    M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
    M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)

    plot_grid.showImage(Lg_img, "Light direction")
    plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")

    title = ""
    plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
    #plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)

    images = []

    for i in xrange(48):
        images.append(figure2numpy(fig))

    for li, L in enumerate(Ls):
        print li
        fig.clear()
        fig.suptitle("Relighting", fontsize=font_size)
        plot_grid.setPlot(1, 1)

        C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)

        C1_lumo = M_lumo.shading(LdotN(L, N_lumo).flatten()).reshape(C0_32F.shape)
        C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
        C1_toon = toon_sfs.relighting(L)

        plot_grid.showImage(lightSphere(L), "Light direction")

        plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "Ground-truth")

        title = ""
        plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "Our result")

        images.append(figure2numpy(fig))

    file_path = shapeResultFile("Relighting", "Relighting_%s_%s" %(shape_name, cmap_id), file_ext=".wmv")
    saveVideo(file_path, images)

def relightingFigure():
    num_methods = 3
    num_lights = 2
    num_rows = num_lights + 1
    num_cols = num_methods + 2

    w = 15
    h = w * num_rows / num_cols

    fig, axes = plt.subplots(figsize=(w, h))
    font_size = 15
    fig.subplots_adjust(left=0.02, right=0.98, top=0.96, bottom=0.04, hspace=0.15, wspace=0.1)
    fig.suptitle("", fontsize=font_size)

    plot_grid = SubplotGrid(num_rows, num_cols)

    Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
    Lg_img = lightSphere(Lg)

    L1 = normalizeVector(np.array([0.3, 0.5, 0.6]))

    # Ls = [normalizeVector(Lg * (1.0 - t) + t * L1) for t in np.linspace(0.0, 1.0, num_lights) ]
    Ls = [normalizeVector(Lg + 0.5 * np.cos(t) * np.array(1, 0, 0) + 0.5 * np.sin(t) * np.array(0, 1, 0)) for t in np.linspace(0.0, 1.0, num_lights) ]

    shape_name = "Vase"

    Ng_data = shapeFile(shape_name)
    Ng_data = loadNormal(Ng_data)
    Ng_32F, A_8U = Ng_data

    N0_file = shapeResultFile(result_name="InitialNormal", data_name=shape_name)
    N0_data = loadNormal(N0_file)
    N0_32F, A_8U = N0_data
    A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)

    cmap_id = 3
    colormap_file = colorMapFile(cmap_id)
    M_32F = loadColorMap(colormap_file)
    C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)

    toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
    toon_sfs.setInitialNormal(N0_32F)
    toon_sfs.setNumIterations(iterations=20)
    toon_sfs.setWeights(w_lap=5.0)
    toon_sfs.run()

    N_toon = toon_sfs.normal()
    C_toon = toon_sfs.shading()

    C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
    C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)

    M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
    M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)

    plot_grid.showImage(Lg_img, "Light direction")
    plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")

    title = ""
    plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
    #plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)

    for L in Ls:
        C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)

        C1_lumo = M_lumo.shading(LdotN(L, N_lumo).flatten()).reshape(C0_32F.shape)
        C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
        C1_toon = toon_sfs.relighting(L)

        plot_grid.showImage(lightSphere(L), "")

        plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "")

        title = ""
        plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "")

    # showMaximize()
    file_path = shapeResultFile("Relighting", "RelightingComparison", file_ext=".png")
    fig.savefig(file_path, transparent=True)

if __name__ == '__main__':
    relightingVideo(shape_name="Lucy", cmap_id=23)<commit_msg>Update batch.<commit_after># -*- coding: utf-8 -*-
## @package ivf.batch.relighting_figure
#
#  ivf.batch.relighting_figure utility package.
#  @author      tody
#  @date        2016/02/22

import numpy as np
import matplotlib.pyplot as plt
import cv2

from ivf.np.norm import normalizeVector
from ivf.plot.window import SubplotGrid, showMaximize
from ivf.datasets.shape import shapeFile, shapeResultFile
from ivf.io_util.image import loadNormal
from ivf.datasets.colormap import colorMapFile, loadColorMap
from ivf.core.shader.toon import ColorMapShader
from ivf.core.sfs.toon_sfs import ToonSFS
from ivf.batch.shape_estimation_figure import lumoSFS, wuSFS, estimatedReflectance
from ivf.core.shader.shader import LdotN
from ivf.cv.image import setAlpha, to32F
from ivf.core.shader.light_sphere import lightSphere
from ivf.plot.fig2np import figure2numpy
from ivf.io_util.video import saveVideo

def relightingVideo(shape_name="Ogre", cmap_id=3):
    num_methods = 3
    num_rows = 1
    num_cols = num_methods + 2

    num_lights = 120

    w = 10
    h = 5

    fig, axes = plt.subplots(figsize=(w, h))
    font_size = 15
    fig.subplots_adjust(left=0.02, right=0.98, top=0.96, bottom=0.04, hspace=0.15, wspace=0.1)
    fig.suptitle("Shading Analysis", fontsize=font_size)

    plot_grid = SubplotGrid(num_rows, num_cols)

    Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
    Lg_img = lightSphere(Lg)

    L1 = normalizeVector(np.array([0.5, 0.5, 0.6]))

    Ls = [normalizeVector(Lg * (1.0 - t) + t * L1) for t in np.linspace(0.0, 1.0, num_lights) ]
    # Ls = [normalizeVector(Lg + 1.0 * np.cos(t) * np.array([1, 0, 0]) + 1.0 * np.sin(t) * np.array([0, 1, 0])) for t in np.linspace(0.0, 1.0, num_lights) ]

    Ng_data = shapeFile(shape_name)
    Ng_data = loadNormal(Ng_data)
    Ng_32F, A_8U = Ng_data

    N0_file = shapeResultFile(result_name="InitialNormal", data_name=shape_name)
    N0_data = loadNormal(N0_file)
    N0_32F, A_8U = N0_data
    A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)

    colormap_file = colorMapFile(cmap_id)
    M_32F = loadColorMap(colormap_file)
    C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)

    toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
    toon_sfs.setInitialNormal(N0_32F)
    toon_sfs.setNumIterations(iterations=50)
    toon_sfs.setWeights(w_lap=0.2)
    toon_sfs.run()

    N_toon = toon_sfs.normal()
    C_toon = toon_sfs.shading()

    C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
    C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)

    M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
    M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)

    plot_grid.showImage(Lg_img, "Light direction")
    plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")

    title = ""
    plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
    #plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)

    images = []

    for i in xrange(48):
        images.append(figure2numpy(fig))

    for li, L in enumerate(Ls):
        print li
        fig.clear()
        fig.suptitle("Relighting", fontsize=font_size)
        plot_grid.setPlot(1, 1)

        C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)

        C1_lumo = M_lumo.shading(LdotN(L, N_lumo).flatten()).reshape(C0_32F.shape)
        C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
        C1_toon = toon_sfs.relighting(L)

        plot_grid.showImage(lightSphere(L), "Light direction")

        plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "Ground-truth")

        title = ""
        plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "Our result")

        images.append(figure2numpy(fig))

    file_path = shapeResultFile("Relighting", "Relighting_%s_%s" %(shape_name, cmap_id), file_ext=".wmv")
    saveVideo(file_path, images)

def relightingFigure(shape_name="Vase", cmap_id=3):
    num_methods = 3
    num_lights = 2
    num_rows = num_lights + 1
    num_cols = num_methods + 2

    w = 15
    h = w * num_rows / num_cols

    fig, axes = plt.subplots(figsize=(w, h))
    font_size = 15
    fig.subplots_adjust(left=0.02, right=0.98, top=0.96, bottom=0.04, hspace=0.15, wspace=0.1)
    fig.suptitle("", fontsize=font_size)

    plot_grid = SubplotGrid(num_rows, num_cols)

    Lg = normalizeVector(np.array([-0.2, 0.3, 0.5]))
    Lg_img = lightSphere(Lg)

    L1 = normalizeVector(np.array([0.0, 0.7, 0.6]))
    L2 = normalizeVector(np.array([0.3, 0.5, 0.6]))

    # Ls = [normalizeVector(Lg * (1.0 - t) + t * L1) for t in np.linspace(0.0, 1.0, num_lights) ]
    Ls = [L1, L2]

    Ng_data = shapeFile(shape_name)
    Ng_data = loadNormal(Ng_data)
    Ng_32F, A_8U = Ng_data

    N0_file = shapeResultFile(result_name="InitialNormal", data_name=shape_name)
    N0_data = loadNormal(N0_file)
    N0_32F, A_8U = N0_data
    A_8U = cv2.bilateralFilter(A_8U, 0, 5, 2)

    colormap_file = colorMapFile(cmap_id)
    M_32F = loadColorMap(colormap_file)
    C0_32F = ColorMapShader(M_32F).diffuseShading(Lg, Ng_32F)

    toon_sfs = ToonSFS(Lg, C0_32F, A_8U)
    toon_sfs.setInitialNormal(N0_32F)
    toon_sfs.setNumIterations(iterations=50)
    toon_sfs.setWeights(w_lap=0.5)
    toon_sfs.run()

    N_toon = toon_sfs.normal()
    C_toon = toon_sfs.shading()

    C_lumo, N_lumo = lumoSFS(C0_32F, Lg, N0_32F, A_8U)
    C_wu, N_wu = wuSFS(C0_32F, Lg, N0_32F, A_8U)

    M_lumo = estimatedReflectance(C0_32F, Lg, N_lumo, A_8U)
    M_wu = estimatedReflectance(C0_32F, Lg, N_wu, A_8U)

    plot_grid.showImage(Lg_img, "Light direction")
    plot_grid.showImage(setAlpha(C0_32F, to32F(A_8U)), "Ground-truth")

    title = ""
    plot_grid.showImage(setAlpha(C_lumo, to32F(A_8U)), "Lumo")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_wu, to32F(A_8U)), "Lambert assumption")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
    plot_grid.showImage(setAlpha(C_toon, to32F(A_8U)), "Our result")
    #plot_grid.showColorMap(C_error_toon, title, v_min=0, v_max=0.1, with_colorbar=True)

    for L in Ls:
        C1 = ColorMapShader(M_32F).diffuseShading(L, Ng_32F)

        C1_lumo = M_lumo.shading(LdotN(L, N_lumo).flatten()).reshape(C0_32F.shape)
        C1_wu = M_wu.shading(LdotN(L, N_wu).flatten()).reshape(C0_32F.shape)
        C1_toon = toon_sfs.relighting(L)

        plot_grid.showImage(lightSphere(L), "")

        plot_grid.showImage(setAlpha(C1, to32F(A_8U)), "")

        title = ""
        plot_grid.showImage(setAlpha(C1_lumo, to32F(A_8U)), "")
    #plot_grid.showColorMap(C_error_lumo, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_wu, to32F(A_8U)), "")
    #plot_grid.showColorMap(C_error_wu, title, v_min=0, v_max=0.1, with_colorbar=True)
        plot_grid.showImage(setAlpha(C1_toon, to32F(A_8U)), "")

    # showMaximize()
    file_path = shapeResultFile("Relighting", "RelightingComparison", file_ext=".png")
    fig.savefig(file_path, transparent=True)

if __name__ == '__main__':
    relightingFigure(shape_name="Ogre", cmap_id=5)<|endoftext|>


<commit_before>"""
Cloud Manager module - the CloudManager class holds all the cloud connectors
for a given group.
"""

import logging

import openstackcloud
import localhostcloud

from cloudscheduler.lib.db_config import Config

class CloudManager():

    """
    CloudManager class for holding a groups resources and their group yaml
    """
    def __init__(self, name, group_resources, group_yamls, metadata):
        """
        Create a new CloudManager.
        :param name: The name of the group
        :param group_resources: sqlalchemy result of this groups resources
        :param group_yamls: the group's yaml from the database.
        """
        self.log = logging.getLogger(__name__)
        self.name = name
        self.clouds = {}
        self.group_resources = group_resources
        self.group_yamls = group_yamls
        self.metadata = metadata
        self.config = Config('/etc/cloudscheduler/cloudscheduler.yaml', [])

    def setup(self):
        """
        Setup the group cloud resources and fetch cloud specific
        yaml from the database.
        """
        self.config.db_open()

        for cloud in self.group_resources:
            try:
                if cloud.cloud_type == 'localhost':
                    newcloud = localhostcloud.LocalHostCloud(resource=cloud,
                                                             metadata=self.metadata[cloud.cloud_name])
                else:
                    newcloud = openstackcloud.\
                        OpenStackCloud(resource=cloud,
                                       metadata=self.metadata[cloud.cloud_name]
                                       if cloud.cloud_name in self.metadata.keys() else None)
                if newcloud:
                    self.clouds[newcloud.name] = newcloud
            except Exception as ex:
                self.log.exception(ex)
        self.log.debug("Added all clouds for group: %s", self.name)
        try:
            self.config.db_close()
        except Exception as ex:
            self.log.exception(ex)
<commit_msg>make sure there's some metadata<commit_after>"""
Cloud Manager module - the CloudManager class holds all the cloud connectors
for a given group.
"""

import logging

import openstackcloud
import localhostcloud

from cloudscheduler.lib.db_config import Config

class CloudManager():

    """
    CloudManager class for holding a groups resources and their group yaml
    """
    def __init__(self, name, group_resources, group_yamls, metadata):
        """
        Create a new CloudManager.
        :param name: The name of the group
        :param group_resources: sqlalchemy result of this groups resources
        :param group_yamls: the group's yaml from the database.
        """
        self.log = logging.getLogger(__name__)
        self.name = name
        self.clouds = {}
        self.group_resources = group_resources
        self.group_yamls = group_yamls
        self.metadata = metadata
        self.config = Config('/etc/cloudscheduler/cloudscheduler.yaml', [])

    def setup(self):
        """
        Setup the group cloud resources and fetch cloud specific
        yaml from the database.
        """
        self.config.db_open()

        for cloud in self.group_resources:
            try:
                if cloud.cloud_type == 'localhost':
                    newcloud = localhostcloud.LocalHostCloud(resource=cloud,
                                                             metadata=self.metadata[cloud.cloud_name])
                else:
                    newcloud = openstackcloud.\
                        OpenStackCloud(resource=cloud,
                                       metadata=self.metadata[cloud.cloud_name]
                                       if self.metadata and cloud.cloud_name in self.metadata.keys()
                                       else None)
                if newcloud:
                    self.clouds[newcloud.name] = newcloud
            except Exception as ex:
                self.log.exception(ex)
        self.log.debug("Added all clouds for group: %s", self.name)
        try:
            self.config.db_close()
        except Exception as ex:
            self.log.exception(ex)
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import testing
from chainer.testing import attr
import chainer.training.updaters.multiprocess_parallel_updater as mpu

import copy

class SimpleNet(chainer.Chain):
    insize = 5

    def __init__(self):
        super(SimpleNet, self).__init__(
            conv=chainer.links.Convolution2D(2, 2, 3),
            fc=chainer.links.Linear(18, 2),
        )
        self.train = True

    def clear(self):
        self.loss = None
        self.accuracy = None

    def __call__(self, x, t):
        h = chainer.functions.relu(self.conv(x))
        y = self.fc(h)

        self.loss = chainer.functions.softmax_cross_entropy(y, t)
        self.accuracy = chainer.functions.accuracy(y, t)

        return self.loss

class TestGatherScatter(unittest.TestCase):

    def setUp(self):
        pass

    @attr.gpu
    def test_gather_scatter_grads(self):
        cupy = cuda.cupy
        model0 = SimpleNet()
        model1 = copy.deepcopy(model0)

        model0.to_gpu()
        model1.to_gpu()

        optimizer0 = chainer.optimizers.SGD(lr=1.0)
        optimizer0.setup(model0)

        optimizer1 = chainer.optimizers.SGD(lr=1.0)
        optimizer1.setup(model1)

        bsize = 8

        x = numpy.random.uniform(0, 1, (bsize, 2, 5, 5)).astype(numpy.float32)
        t = numpy.empty(bsize, dtype=numpy.int32)
        for i in range(bsize):
            t[i] = i % 2

        x = chainer.Variable(chainer.cuda.to_gpu(x))
        t = chainer.Variable(chainer.cuda.to_gpu(t))

        loss0 = model0(x, t)
        model1(x, t)

        model0.cleargrads()
        model1.cleargrads()

        loss0.backward()
        gg0 = mpu.gather_grads(model0)
        mpu.scatter_grads(model1, gg0)

        cupy.testing.assert_array_equal(model0.conv.W.grad, model1.conv.W.grad)
        cupy.testing.assert_array_equal(model0.conv.b.grad, model1.conv.b.grad)
        cupy.testing.assert_array_equal(model0.fc.W.grad, model1.fc.W.grad)
        cupy.testing.assert_array_equal(model0.fc.b.grad, model1.fc.b.grad)

        optimizer0.update()
        optimizer1.update()

        cupy.testing.assert_array_equal(model0.conv.W.data, model1.conv.W.data)
        cupy.testing.assert_array_equal(model0.conv.b.data, model1.conv.b.data)
        cupy.testing.assert_array_equal(model0.fc.W.data, model1.fc.W.data)
        cupy.testing.assert_array_equal(model0.fc.b.data, model1.fc.b.data)

    def test_gather_grads_raise_on_cpu(self):
        model = SimpleNet()
        with self.assertRaises(RuntimeError):
            mpu.gather_grads(model)

    @attr.gpu
    def test_gather_scatter_params(self):
        cupy = cuda.cupy
        model0 = SimpleNet()
        model1 = SimpleNet()

        model0.to_gpu()
        model1.to_gpu()

        gp0 = mpu.gather_params(model0)
        mpu.scatter_params(model1, gp0)

        cupy.testing.assert_array_equal(model0.conv.W.data, model1.conv.W.data)
        cupy.testing.assert_array_equal(model0.conv.b.data, model1.conv.b.data)
        cupy.testing.assert_array_equal(model0.fc.W.data, model1.fc.W.data)
        cupy.testing.assert_array_equal(model0.fc.b.data, model1.fc.b.data)

    def test_gather_params_raise_on_cpu(self):
        model = SimpleNet()
        with self.assertRaises(RuntimeError):
            mpu.gather_params(model)

testing.run_module(__name__, __file__)
<commit_msg>Remove redundant code<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import testing
from chainer.testing import attr
import chainer.training.updaters.multiprocess_parallel_updater as mpu

import copy

class SimpleNet(chainer.Chain):
    insize = 5

    def __init__(self):
        super(SimpleNet, self).__init__(
            conv=chainer.links.Convolution2D(2, 2, 3),
            fc=chainer.links.Linear(18, 2),
        )
        self.train = True

    def clear(self):
        self.loss = None
        self.accuracy = None

    def __call__(self, x, t):
        h = chainer.functions.relu(self.conv(x))
        y = self.fc(h)

        self.loss = chainer.functions.softmax_cross_entropy(y, t)
        self.accuracy = chainer.functions.accuracy(y, t)

        return self.loss

class TestGatherScatter(unittest.TestCase):

    def setUp(self):
        pass

    @attr.gpu
    def test_gather_scatter_grads(self):
        cupy = cuda.cupy
        model0 = SimpleNet()
        model1 = copy.deepcopy(model0)

        model0.to_gpu()
        model1.to_gpu()

        optimizer0 = chainer.optimizers.SGD(lr=1.0)
        optimizer0.setup(model0)

        optimizer1 = chainer.optimizers.SGD(lr=1.0)
        optimizer1.setup(model1)

        bsize = 8

        x = numpy.random.uniform(0, 1, (bsize, 2, 5, 5)).astype(numpy.float32)
        t = numpy.empty(bsize, dtype=numpy.int32)
        for i in range(bsize):
            t[i] = i % 2

        x = chainer.Variable(chainer.cuda.to_gpu(x))
        t = chainer.Variable(chainer.cuda.to_gpu(t))

        loss0 = model0(x, t)

        model0.cleargrads()
        model1.cleargrads()

        loss0.backward()
        gg0 = mpu.gather_grads(model0)
        mpu.scatter_grads(model1, gg0)

        cupy.testing.assert_array_equal(model0.conv.W.grad, model1.conv.W.grad)
        cupy.testing.assert_array_equal(model0.conv.b.grad, model1.conv.b.grad)
        cupy.testing.assert_array_equal(model0.fc.W.grad, model1.fc.W.grad)
        cupy.testing.assert_array_equal(model0.fc.b.grad, model1.fc.b.grad)

        optimizer0.update()
        optimizer1.update()

        cupy.testing.assert_array_equal(model0.conv.W.data, model1.conv.W.data)
        cupy.testing.assert_array_equal(model0.conv.b.data, model1.conv.b.data)
        cupy.testing.assert_array_equal(model0.fc.W.data, model1.fc.W.data)
        cupy.testing.assert_array_equal(model0.fc.b.data, model1.fc.b.data)

    def test_gather_grads_raise_on_cpu(self):
        model = SimpleNet()
        with self.assertRaises(RuntimeError):
            mpu.gather_grads(model)

    @attr.gpu
    def test_gather_scatter_params(self):
        cupy = cuda.cupy
        model0 = SimpleNet()
        model1 = SimpleNet()

        model0.to_gpu()
        model1.to_gpu()

        gp0 = mpu.gather_params(model0)
        mpu.scatter_params(model1, gp0)

        cupy.testing.assert_array_equal(model0.conv.W.data, model1.conv.W.data)
        cupy.testing.assert_array_equal(model0.conv.b.data, model1.conv.b.data)
        cupy.testing.assert_array_equal(model0.fc.W.data, model1.fc.W.data)
        cupy.testing.assert_array_equal(model0.fc.b.data, model1.fc.b.data)

    def test_gather_params_raise_on_cpu(self):
        model = SimpleNet()
        with self.assertRaises(RuntimeError):
            mpu.gather_params(model)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''whatlastgenre genretag'''

from __future__ import division, print_function

import ConfigParser
import StringIO
from collections import defaultdict
import difflib
import itertools
import logging
import math
import pkgutil
import re

LOG = logging.getLogger('whatlastgenre')

class GenreTags(object):
    '''Class for managing genre tags.'''

    def __init__(self, conf):
        self.conf = conf
        self.tags = None
        # list activated filters
        filters = ['badtags', 'generic', 'name']
        filters += conf.get_list('genres', 'filters')
        # get and validate tagsfile
        self.tagsfile = self.parse_tagsfile(filters)
        # fill matchlist
        self.matchlist = self.tagsfile.options('basictags')
        self.matchlist += conf.get_list('genres', 'love')
        self.matchlist += conf.get_list('genres', 'hate')
        self.matchlist += conf.get_list('genres', 'blacklist')
        # fill replaces dict
        self.replaces = {}
        for pattern, repl in self.tagsfile.items('replaceme', True):
            self.replaces.update({pattern: repl})
        # build filter
        filter_ = conf.get_list('genres', 'blacklist')
        for sec in [s for s in self.tagsfile.sections()
                    if s.startswith('filter_')]:
            if sec[7:] in filters:
                filter_ += self.tagsfile.options(sec)
            elif sec.endswith('_fuzzy') and sec[7:-6] in filters:
                for tag in self.tagsfile.options(sec):
                    filter_.append('.*%s.*' % tag)
        # set up regex
        self.regex = {}
        # compile some config options and tagsfile sections
        for sec, pats in ([(s, conf.get_list('genres', s))
                           for s in ['love', 'hate']] +
                          [(s, self.tagsfile.options(s))
                           for s in ['uppercase', 'dontsplit', 'replaceme']]):
            self.regex[sec] = re.compile('(%s)$' % '|'.join(pats), re.I)
        # compile filter in chunks
        self.regex['filter'] = []
        for i in range(0, len(filter_), 384):
            pat = '(%s)$' % '|'.join(filter_[i:i + 384])
            self.regex['filter'].append(re.compile(pat, re.I))

    def _add(self, group, name, score):
        '''Adds a tag with a name and score to a group.

        After some filter, replace, match, split and score,
        True is returned if the tag was added, False otherwise.

        :param group: tag group (for different scores on merging later)
        :param name: the name of the tag to add
        :param score: the score of the tag to add
        '''
        if not score:
            return False
        name = name.encode('ascii', 'ignore').lower()
        name = self._replace(name)
        if self._filter(name):
            return False
        name = self._match(name)
        score = self._split(group, name, score)
        if not score:
            return False
        self.tags[group][name] += score
        return True

    def _replace(self, name):
        '''Applies all the replaces to a tag name and returns it.

        Uses a precompiled search pattern of all replace patterns to identify
        tag names that will get some replacement for increased performance.
        '''
        if self.regex['replaceme'].search(name):
            for pattern, repl in self.replaces.items():
                name = re.sub(pattern, repl, name, 0, re.I)
        return re.sub('(_| +)', ' ', name).strip()

    def _filter(self, name):
        '''Filters a tag by name, returns True if tag is filtered.'''
        if len(name) < 3 or len(name) > 19:
            return True
        if re.search(r'[^a-z0-9&\-_/\\,;\.\+\* ]', name, re.I):
            return True
        if self.regex['filter_album'].match(name):
            return True
        if any(f.match(name) for f in self.regex['filter']):
            return True
        return False

    def _match(self, name):
        '''Matches a tag name with known tag names.'''
        mli = []
        for taglist in self.tags.values():
            mli += taglist.keys()
        mli += self.matchlist
        # next two lines should improve performance
        if name in mli:
            return name
        # don't change cutoff, add replaces instead
        match = difflib.get_close_matches(name, mli, 1, .8572)
        if match:
            return match[0].lower()
        return name

    def _split(self, group, name, score):
        '''Splits a tag and adds its parts with modified score,
        returns the remaining score for the base tag.'''
        if self.regex['dontsplit'].match(name):
            return score
        name = re.sub(r'([_/\\,;\.\+\*]| and )', '&', name, 0, re.I)
        if '&' in name:
            for part in [p for p in name.split('&') if not self._filter(p)]:
                self._add(group, part, score)
                return None
        elif ' ' in name.strip():
            rawparts = name.split(' ')
            parts = [p for p in rawparts if not self._filter(p)]
            if not parts:
                return None
            parts = itertools.combinations(parts, max(1, len(parts) - 1))
            for part in set(parts):
                self._add(group, ' '.join(part), score)
            if len(rawparts) > 2:
                return None
            return score * self.conf.getfloat('scores', 'splitup')
        return score

    def reset(self, pattern):
        '''Resets the genre tags dict and sets a new album filter pattern.

        :param pattern: compiled album filter match pattern
        '''
        self.tags = {'artist': defaultdict(float), 'album': defaultdict(float)}
        self.regex['filter_album'] = pattern

    def add(self, source, group, tags):
        '''Adds multiple tags from a source to a group.

        Tags can be with counts (as dict) or without counts (as list). The tag
        scores get multiplied with the source score modifier of the
        corresponding source. Returns the number of tags added. Note that tags
        that get split later still count as one tag, no matter how many parts
        came from it.

        :param source: the source where the tags came from
        :param group: the group where the tags get added to
        :param tags: the tags with (as dict) or without (as list) counts
        '''
        if not tags:
            return 0
        added = 0
        multi = self.conf.getfloat('scores', 'src_%s' % source)
        if isinstance(tags, dict):
            max_ = max(tags.values())
            if max_ == 0:
                return 0
            for key, val in sorted(tags.items(), key=tags.get, reverse=1)[:99]:
                if self._add(group, key, val / max_ * multi):
                    added += 1
        elif isinstance(tags, list):
            score = .85 ** (len(tags) - 1)
            for name in tags:
                if self._add(group, name, score * multi):
                    added += 1
        return added

    def get(self, various=False):
        '''Merges all tag groups and returns the sorted and formated genres.'''
        for group, tags in ((k, v) for k, v in self.tags.items() if v):
            # norm tag scores
            max_ = max(tags.values())
            for tag, score in tags.items():
                tags[tag] = score / max_
            # verbose output
            tags = [(self.format(k), v) for k, v in sorted
                    (tags.items(), key=lambda (k, v): (v, k), reverse=1)
                    if v > 0.1]
            tagout = tags if LOG.level == logging.DEBUG else tags[:12]
            tagout = self.tagprintstr(tagout, "%5.2f %-19s")
            LOG.info("Best %6s genres (%d):\n%s", group, len(tags), tagout)
        # merge artist and album genres
        genres = defaultdict(float)
        for group, tags in self.tags.items():
            mult = 1
            if group == 'artist':
                mult = 'various' if various else 'artist'
                mult = self.conf.getfloat('scores', mult)
            for tag, score in tags.items():
                score = score * mult
                score *= 2 if self.regex['love'].match(tag) else 1
                score *= 0.5 if self.regex['hate'].match(tag) else 1
                if score > 0.1:
                    genres[tag] += score
        # format genres
        genres = {self.format(k): v for k, v in genres.items()}
        # sort and return keys
        return sorted(genres, key=genres.get, reverse=1)

    def format(self, name):
        '''Formats a tag to correct case.'''
        split = name.split(' ')
        for i in range(len(split)):
            if len(split[i]) < 3 and split[i] != 'nu' or \
                    self.regex['uppercase'].match(split[i]):
                split[i] = split[i].upper()
            elif re.match('[0-9]{4}s', name, re.I):
                split[i] = split[i].lower()
            else:
                split[i] = split[i].title()
        return ' '.join(split)

    @classmethod
    def tagprintstr(cls, tags, pattern):
        '''Returns a string of tags formated with pattern in 3 columns.

        :param tags: dict of tag name with scores
        :param pattern: should not exceed (80-2)/3 = 26 chars length.
        '''
        lines = []
        num = int(math.ceil(len(tags) / 3))
        for i in range(num):
            row = []
            for j in [j for j in range(3) if i + num * j < len(tags)]:
                col = pattern % (tags[i + num * j][1], tags[i + num * j][0])
                row.append(col)
            lines.append(' '.join(row))
        return '\n'.join(lines)

    @classmethod
    def parse_tagsfile(cls, filters):
        '''Returns a ConfigParser object for the tagsfile.

        :param filters: list of filters to check for existence
        '''
        tagsfilestr = pkgutil.get_data('wlg', 'tags.txt')
        parser = ConfigParser.SafeConfigParser(allow_no_value=True)
        parser.readfp(StringIO.StringIO(tagsfilestr))
        # tags file validation
        for sec in [s for s in ['basictags', 'uppercase', 'dontsplit',
                                'replaceme']
                    if not parser.has_section(s)]:
            print("Got no [%s] from tag.txt file." % sec)
            exit()
        for sec in [s for s in filters if
                    not parser.has_section('filter_%s' % s) and
                    not parser.has_section('filter_%s_fuzzy' % s)]:
            print("The configured filter '%s' doesn't has a [filter_%s[_fuzzy]]"
                  " section in the tags.txt file." % (sec, sec))
            exit()
        return parser

<commit_msg>proper handling of tags with 0 as count<commit_after>#!/usr/bin/env python
'''whatlastgenre genretag'''

from __future__ import division, print_function

import ConfigParser
import StringIO
from collections import defaultdict
import difflib
import itertools
import logging
import math
import pkgutil
import re

LOG = logging.getLogger('whatlastgenre')

class GenreTags(object):
    '''Class for managing genre tags.'''

    def __init__(self, conf):
        self.conf = conf
        self.tags = None
        # list activated filters
        filters = ['badtags', 'generic', 'name']
        filters += conf.get_list('genres', 'filters')
        # get and validate tagsfile
        self.tagsfile = self.parse_tagsfile(filters)
        # fill matchlist
        self.matchlist = self.tagsfile.options('basictags')
        self.matchlist += conf.get_list('genres', 'love')
        self.matchlist += conf.get_list('genres', 'hate')
        self.matchlist += conf.get_list('genres', 'blacklist')
        # fill replaces dict
        self.replaces = {}
        for pattern, repl in self.tagsfile.items('replaceme', True):
            self.replaces.update({pattern: repl})
        # build filter
        filter_ = conf.get_list('genres', 'blacklist')
        for sec in [s for s in self.tagsfile.sections()
                    if s.startswith('filter_')]:
            if sec[7:] in filters:
                filter_ += self.tagsfile.options(sec)
            elif sec.endswith('_fuzzy') and sec[7:-6] in filters:
                for tag in self.tagsfile.options(sec):
                    filter_.append('.*%s.*' % tag)
        # set up regex
        self.regex = {}
        # compile some config options and tagsfile sections
        for sec, pats in ([(s, conf.get_list('genres', s))
                           for s in ['love', 'hate']] +
                          [(s, self.tagsfile.options(s))
                           for s in ['uppercase', 'dontsplit', 'replaceme']]):
            self.regex[sec] = re.compile('(%s)$' % '|'.join(pats), re.I)
        # compile filter in chunks
        self.regex['filter'] = []
        for i in range(0, len(filter_), 384):
            pat = '(%s)$' % '|'.join(filter_[i:i + 384])
            self.regex['filter'].append(re.compile(pat, re.I))

    def _add(self, group, name, score):
        '''Adds a tag with a name and score to a group.

        After some filter, replace, match, split and score,
        True is returned if the tag was added, False otherwise.

        :param group: tag group (for different scores on merging later)
        :param name: the name of the tag to add
        :param score: the score of the tag to add
        '''
        if not score:
            return False
        name = name.encode('ascii', 'ignore').lower()
        name = self._replace(name)
        if self._filter(name):
            return False
        name = self._match(name)
        score = self._split(group, name, score)
        if not score:
            return False
        self.tags[group][name] += score
        return True

    def _replace(self, name):
        '''Applies all the replaces to a tag name and returns it.

        Uses a precompiled search pattern of all replace patterns to identify
        tag names that will get some replacement for increased performance.
        '''
        if self.regex['replaceme'].search(name):
            for pattern, repl in self.replaces.items():
                name = re.sub(pattern, repl, name, 0, re.I)
        return re.sub('(_| +)', ' ', name).strip()

    def _filter(self, name):
        '''Filters a tag by name, returns True if tag is filtered.'''
        if len(name) < 3 or len(name) > 19:
            return True
        if re.search(r'[^a-z0-9&\-_/\\,;\.\+\* ]', name, re.I):
            return True
        if self.regex['filter_album'].match(name):
            return True
        if any(f.match(name) for f in self.regex['filter']):
            return True
        return False

    def _match(self, name):
        '''Matches a tag name with known tag names.'''
        mli = []
        for taglist in self.tags.values():
            mli += taglist.keys()
        mli += self.matchlist
        # next two lines should improve performance
        if name in mli:
            return name
        # don't change cutoff, add replaces instead
        match = difflib.get_close_matches(name, mli, 1, .8572)
        if match:
            return match[0].lower()
        return name

    def _split(self, group, name, score):
        '''Splits a tag and adds its parts with modified score,
        returns the remaining score for the base tag.'''
        if self.regex['dontsplit'].match(name):
            return score
        name = re.sub(r'([_/\\,;\.\+\*]| and )', '&', name, 0, re.I)
        if '&' in name:
            for part in [p for p in name.split('&') if not self._filter(p)]:
                self._add(group, part, score)
                return None
        elif ' ' in name.strip():
            rawparts = name.split(' ')
            parts = [p for p in rawparts if not self._filter(p)]
            if not parts:
                return None
            parts = itertools.combinations(parts, max(1, len(parts) - 1))
            for part in set(parts):
                self._add(group, ' '.join(part), score)
            if len(rawparts) > 2:
                return None
            return score * self.conf.getfloat('scores', 'splitup')
        return score

    def reset(self, pattern):
        '''Resets the genre tags dict and sets a new album filter pattern.

        :param pattern: compiled album filter match pattern
        '''
        self.tags = {'artist': defaultdict(float), 'album': defaultdict(float)}
        self.regex['filter_album'] = pattern

    def add(self, source, group, tags):
        '''Adds multiple tags from a source to a group.

        Tags can be with counts (as dict) or without counts (as list). The tag
        scores get multiplied with the source score modifier of the
        corresponding source. Returns the number of tags added. Note that tags
        that get split later still count as one tag, no matter how many parts
        came from it.

        :param source: the source where the tags came from
        :param group: the group where the tags get added to
        :param tags: the tags with (as dict) or without (as list) counts
        '''
        if not tags:
            return 0
        added = 0
        multi = self.conf.getfloat('scores', 'src_%s' % source)
        if isinstance(tags, dict):
            max_ = max(tags.values())
            if not max_ or (x for x in tags.values() if not x):
                # handle without counts anyway
                return self.add(source, group, tags.keys())
            for key, val in sorted(tags.items(), key=tags.get, reverse=1)[:99]:
                if self._add(group, key, val / max_ * multi):
                    added += 1
        elif isinstance(tags, list):
            score = .85 ** (len(tags) - 1)
            for name in tags:
                if self._add(group, name, score * multi):
                    added += 1
        return added

    def get(self, various=False):
        '''Merges all tag groups and returns the sorted and formated genres.'''
        for group, tags in ((k, v) for k, v in self.tags.items() if v):
            # norm tag scores
            max_ = max(tags.values())
            for tag, score in tags.items():
                tags[tag] = score / max_
            # verbose output
            tags = [(self.format(k), v) for k, v in sorted
                    (tags.items(), key=lambda (k, v): (v, k), reverse=1)
                    if v > 0.1]
            tagout = tags if LOG.level == logging.DEBUG else tags[:12]
            tagout = self.tagprintstr(tagout, "%5.2f %-19s")
            LOG.info("Best %6s genres (%d):\n%s", group, len(tags), tagout)
        # merge artist and album genres
        genres = defaultdict(float)
        for group, tags in self.tags.items():
            mult = 1
            if group == 'artist':
                mult = 'various' if various else 'artist'
                mult = self.conf.getfloat('scores', mult)
            for tag, score in tags.items():
                score = score * mult
                score *= 2 if self.regex['love'].match(tag) else 1
                score *= 0.5 if self.regex['hate'].match(tag) else 1
                if score > 0.1:
                    genres[tag] += score
        # format genres
        genres = {self.format(k): v for k, v in genres.items()}
        # sort and return keys
        return sorted(genres, key=genres.get, reverse=1)

    def format(self, name):
        '''Formats a tag to correct case.'''
        split = name.split(' ')
        for i in range(len(split)):
            if len(split[i]) < 3 and split[i] != 'nu' or \
                    self.regex['uppercase'].match(split[i]):
                split[i] = split[i].upper()
            elif re.match('[0-9]{4}s', name, re.I):
                split[i] = split[i].lower()
            else:
                split[i] = split[i].title()
        return ' '.join(split)

    @classmethod
    def tagprintstr(cls, tags, pattern):
        '''Returns a string of tags formated with pattern in 3 columns.

        :param tags: dict of tag name with scores
        :param pattern: should not exceed (80-2)/3 = 26 chars length.
        '''
        lines = []
        num = int(math.ceil(len(tags) / 3))
        for i in range(num):
            row = []
            for j in [j for j in range(3) if i + num * j < len(tags)]:
                col = pattern % (tags[i + num * j][1], tags[i + num * j][0])
                row.append(col)
            lines.append(' '.join(row))
        return '\n'.join(lines)

    @classmethod
    def parse_tagsfile(cls, filters):
        '''Returns a ConfigParser object for the tagsfile.

        :param filters: list of filters to check for existence
        '''
        tagsfilestr = pkgutil.get_data('wlg', 'tags.txt')
        parser = ConfigParser.SafeConfigParser(allow_no_value=True)
        parser.readfp(StringIO.StringIO(tagsfilestr))
        # tags file validation
        for sec in [s for s in ['basictags', 'uppercase', 'dontsplit',
                                'replaceme']
                    if not parser.has_section(s)]:
            print("Got no [%s] from tag.txt file." % sec)
            exit()
        for sec in [s for s in filters if
                    not parser.has_section('filter_%s' % s) and
                    not parser.has_section('filter_%s_fuzzy' % s)]:
            print("The configured filter '%s' doesn't has a [filter_%s[_fuzzy]]"
                  " section in the tags.txt file." % (sec, sec))
            exit()
        return parser

<|endoftext|>