<commit_before># This file is part of Bika LIMS
#
# Copyright 2011-2016 by it's authors.
# Some rights reserved. See LICENSE.txt, AUTHORS.txt.

from AccessControl import getSecurityManager
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t, dicts_to_dict
from bika.lims.browser.bika_listing import BikaListingView
from bika.lims.browser.sample import SamplePartitionsView
from bika.lims.content.analysisrequest import schema as AnalysisRequestSchema
from bika.lims.permissions import *
from bika.lims.utils import logged_in_client
from bika.lims.utils import to_utf8
from bika.lims.workflow import doActionFor
from DateTime import DateTime
from Products.Archetypes import PloneMessageFactory as PMF
from plone.app.content.browser.interfaces import IFolderContentsView
from plone.app.layout.globals.interfaces import IViewView
from Products.CMFCore.utils import getToolByName
from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
from zope.i18n.locales import locales
from zope.interface import implements

import json
import plone

class AnalysisRequestAnalysesView(BikaListingView):
    implements(IFolderContentsView, IViewView)

    template = ViewPageTemplateFile("templates/analysisrequest_analyses.pt")

    def __init__(self, context, request):
        super(AnalysisRequestAnalysesView, self).__init__(context, request)
        self.catalog = "bika_setup_catalog"
        self.contentFilter = {'portal_type': 'AnalysisService',
                              'sort_on': 'sortable_title',
                              'inactive_state': 'active', }
        self.context_actions = {}
        self.icon = self.portal_url + "/++resource++bika.lims.images/analysisrequest_big.png"
        self.title = self.context.Title()
        self.show_sort_column = False
        self.show_select_row = False
        self.show_select_column = True
        self.table_only = True
        self.show_select_all_checkbox = False
        self.pagesize = 999999

        self.categories = []
        self.do_cats = self.context.bika_setup.getCategoriseAnalysisServices()
        if self.do_cats:
            self.show_categories = True
            self.expand_all_categories = False
            self.ajax_categories = True
            self.category_index = 'getCategoryTitle'

        self.columns = {
            'Title': {'title': _('Service'),
                      'index': 'sortable_title',
                      'sortable': False, },
            'Hidden': {'title': _('Hidden'),
                       'sortable': False,
                       'type': 'boolean', },
            'Price': {'title': _('Price'),
                      'sortable': False, },
            'Priority': {'title': _('Priority'),
                         'sortable': False,
                         'index': 'Priority',
                         'toggle': True },
            'Partition': {'title': _('Partition'),
                          'sortable': False, },
            'min': {'title': _('Min')},
            'max': {'title': _('Max')},
            'error': {'title': _('Permitted Error %')},
        }

        columns = ['Title', 'Hidden', ]
        ShowPrices = self.context.bika_setup.getShowPrices()
        if ShowPrices:
            columns.append('Price')
            columns.append('Priority')
        ShowPartitions = self.context.bika_setup.getShowPartitions()
        if ShowPartitions:
            columns.append('Partition')
        EnableARSpecs = self.context.bika_setup.getEnableARSpecs()
        if EnableARSpecs:
            columns.append('min')
            columns.append('max')
            columns.append('error')

        self.review_states = [
            {'id': 'default',
             'title': _('All'),
             'contentFilter': {},
             'columns': columns,
             'transitions': [{'id': 'empty'}, ],  # none
             'custom_actions': [{'id': 'save_analyses_button',
                                 'title': _('Save')}, ],
             },
        ]

        # Create Partitions View for this ARs sample
        sample = self.context.getSample()
        p = SamplePartitionsView(sample, self.request)
        p.table_only = True
        p.allow_edit = False
        p.form_id = "parts"
        p.show_select_column = False
        p.show_table_footer = False
        p.review_states[0]['transitions'] = [{'id': 'empty'}, ]  # none
        p.review_states[0]['custom_actions'] = []
        p.review_states[0]['columns'] = ['PartTitle',
                                         'getContainer',
                                         'getPreservation',
                                         'state_title']

        self.parts = p.contents_table()

    def getResultsRange(self):
        """Return the AR Specs sorted by Service UID, so that the JS can
        work easily with the values.
        """
        bsc = self.bika_setup_catalog
        rr_dict_by_service_uid = {}
        rr = self.context.getResultsRange()
        for r in rr:
            keyword = r['keyword']
            try:
                service_uid = bsc(portal_type='AnalysisService',
                                  getKeyword=keyword)[0].UID
                rr_dict_by_service_uid[service_uid] = r
            except IndexError:
                from bika.lims import logger
                error = "No Analysis Service found for Keyword '%s'. "\
                        "Related: LIMS-1614"
                logger.exception(error, keyword)

        return json.dumps(rr_dict_by_service_uid)

    def get_spec_from_ar(self, ar, keyword):
        empty = {'min': '', 'max': '', 'error': '', 'keyword':keyword}
        spec = ar.getResultsRange()
        if spec:
            return dicts_to_dict(spec, 'keyword').get(keyword, empty)
        return empty

    def folderitems(self):
        self.categories = []

        analyses = self.context.getAnalyses(full_objects=True)
        self.analyses = dict([(a.getServiceUID(), a) for a in analyses])
        self.selected = self.analyses.keys()
        self.show_categories = \
            self.context.bika_setup.getCategoriseAnalysisServices()
        self.expand_all_categories = False

        wf = getToolByName(self.context, 'portal_workflow')
        mtool = getToolByName(self.context, 'portal_membership')

        self.allow_edit = mtool.checkPermission('Modify portal content',
                                                self.context)

        items = BikaListingView.folderitems(self)
        analyses = self.context.getAnalyses(full_objects=True)

        parts = self.context.getSample().objectValues('SamplePartition')
        partitions = [{'ResultValue': o.Title(),
                       'ResultText': o.getId()}
                      for o in parts
                      if wf.getInfoFor(o, 'cancellation_state', '') == 'active']
        for x in range(len(items)):
            if not 'obj' in items[x]:
                continue
            obj = items[x]['obj']

            cat = obj.getCategoryTitle()
            items[x]['category'] = cat
            if cat not in self.categories:
                self.categories.append(cat)

            items[x]['selected'] = items[x]['uid'] in self.selected

            items[x]['class']['Title'] = 'service_title'

            # js checks in row_data if an analysis may be removed.
            row_data = {}
            # keyword = obj.getKeyword()
            # if keyword in review_states.keys() \
            #    and review_states[keyword] not in ['sample_due',
            #                                       'to_be_sampled',
            #                                       'to_be_preserved',
            #                                       'sample_received',
            #                                       ]:
            #     row_data['disabled'] = True
            items[x]['row_data'] = json.dumps(row_data)

            calculation = obj.getCalculation()
            items[x]['Calculation'] = calculation and calculation.Title()

            locale = locales.getLocale('en')
            currency = self.context.bika_setup.getCurrency()
            symbol = locale.numbers.currencies[currency].symbol
            items[x]['before']['Price'] = symbol
            items[x]['Price'] = obj.getPrice()
            items[x]['class']['Price'] = 'nowrap'
            items[x]['Priority'] = ''

            if items[x]['selected']:
                items[x]['allow_edit'] = ['Partition', 'min', 'max', 'error']
                if not logged_in_client(self.context):
                    items[x]['allow_edit'].append('Price')

            items[x]['required'].append('Partition')
            items[x]['choices']['Partition'] = partitions

            if obj.UID() in self.analyses:
                analysis = self.analyses[obj.UID()]
                part = analysis.getSamplePartition()
                part = part and part or obj
                items[x]['Partition'] = part.Title()
                spec = self.get_spec_from_ar(self.context,
                                             analysis.getService().getKeyword())
                items[x]["min"] = spec.get("min",'')
                items[x]["max"] = spec.get("max",'')
                items[x]["error"] = spec.get("error",'')
                # Add priority premium
                items[x]['Price'] = analysis.getPrice()
                priority = analysis.getPriority()
                items[x]['Priority'] = priority and priority.Title() or ''
            else:
                items[x]['Partition'] = ''
                items[x]["min"] = ''
                items[x]["max"] = ''
                items[x]["error"] = ''
                items[x]["Priority"] = ''

            after_icons = ''
            if obj.getAccredited():
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/accredited.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Accredited"))
                )
            if obj.getReportDryMatter():
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/dry.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Can be reported as dry matter"))
                )
            if obj.getAttachmentOption() == 'r':
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/attach_reqd.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Attachment required"))
                )
            if obj.getAttachmentOption() == 'n':
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/attach_no.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_('Attachment not permitted'))
                )
            if after_icons:
                items[x]['after']['Title'] = after_icons

            # Display analyses for this Analysis Service in results?
            ser = self.context.getAnalysisServiceSettings(obj.UID())
            items[x]['allow_edit'] = ['Hidden', ]
            items[x]['Hidden'] = ser.get('hidden', obj.getHidden())

        self.categories.sort()
        return items
<commit_msg>analysis request - manage analysis<commit_after># This file is part of Bika LIMS
#
# Copyright 2011-2016 by it's authors.
# Some rights reserved. See LICENSE.txt, AUTHORS.txt.

from AccessControl import getSecurityManager
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t, dicts_to_dict
from bika.lims.browser.bika_listing import BikaListingView
from bika.lims.browser.sample import SamplePartitionsView
from bika.lims.content.analysisrequest import schema as AnalysisRequestSchema
from bika.lims.permissions import *
from bika.lims.utils import logged_in_client
from bika.lims.utils import to_utf8
from bika.lims.workflow import doActionFor
from DateTime import DateTime
from Products.Archetypes import PloneMessageFactory as PMF
from plone.app.content.browser.interfaces import IFolderContentsView
from plone.app.layout.globals.interfaces import IViewView
from Products.CMFCore.utils import getToolByName
from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
from zope.i18n.locales import locales
from zope.interface import implements

import json
import plone

class AnalysisRequestAnalysesView(BikaListingView):
    implements(IFolderContentsView, IViewView)

    template = ViewPageTemplateFile("templates/analysisrequest_analyses.pt")

    def __init__(self, context, request):
        super(AnalysisRequestAnalysesView, self).__init__(context, request)
        self.catalog = "bika_setup_catalog"
        self.contentFilter = {'portal_type': 'AnalysisService',
                              'sort_on': 'sortable_title',
                              'inactive_state': 'active', }
        self.context_actions = {}
        self.icon = self.portal_url + "/++resource++bika.lims.images/analysisrequest_big.png"
        self.title = self.context.Title()
        self.show_sort_column = False
        self.show_select_row = False
        self.show_select_column = True
        self.table_only = True
        self.show_select_all_checkbox = False
        self.pagesize = 999999

        self.categories = []
        self.do_cats = self.context.bika_setup.getCategoriseAnalysisServices()
        if self.do_cats:
            self.show_categories = True
            self.expand_all_categories = False
            self.ajax_categories = True
            self.category_index = 'getCategoryTitle'

        self.columns = {
            'Title': {'title': _('Service'),
                      'index': 'sortable_title',
                      'sortable': False, },
            'Hidden': {'title': _('Hidden'),
                       'sortable': False,
                       'type': 'boolean', },
            'Price': {'title': _('Price'),
                      'sortable': False, },
            'Priority': {'title': _('Priority'),
                         'sortable': False,
                         'index': 'Priority',
                         'toggle': True },
            'Partition': {'title': _('Partition'),
                          'sortable': False, },
            'min': {'title': _('Min')},
            'max': {'title': _('Max')},
            'error': {'title': _('Permitted Error %')},
        }

        columns = ['Title', 'Hidden', ]
        ShowPrices = self.context.bika_setup.getShowPrices()
        if ShowPrices:
            columns.append('Price')
            columns.append('Priority')
        ShowPartitions = self.context.bika_setup.getShowPartitions()
        if ShowPartitions:
            columns.append('Partition')
        EnableARSpecs = self.context.bika_setup.getEnableARSpecs()
        if EnableARSpecs:
            columns.append('min')
            columns.append('max')
            columns.append('error')

        self.review_states = [
            {'id': 'default',
             'title': _('All'),
             'contentFilter': {},
             'columns': columns,
             'transitions': [{'id': 'empty'}, ],  # none
             'custom_actions': [{'id': 'save_analyses_button',
                                 'title': _('Save')}, ],
             },
        ]

        # Create Partitions View for this ARs sample
        sample = self.context.getSample()
        p = SamplePartitionsView(sample, self.request)
        p.table_only = True
        p.allow_edit = False
        p.form_id = "parts"
        p.show_select_column = False
        p.show_table_footer = False
        p.review_states[0]['transitions'] = [{'id': 'empty'}, ]  # none
        p.review_states[0]['custom_actions'] = []
        p.review_states[0]['columns'] = ['PartTitle',
                                         'getContainer',
                                         'getPreservation',
                                         'state_title']

        self.parts = p.contents_table()

    def getResultsRange(self):
        """Return the AR Specs sorted by Service UID, so that the JS can
        work easily with the values.
        """
        bsc = self.bika_setup_catalog
        rr_dict_by_service_uid = {}
        rr = self.context.getResultsRange()
        for r in rr:
            keyword = r['keyword']
            try:
                service_uid = bsc(portal_type='AnalysisService',
                                  getKeyword=keyword)[0].UID
                rr_dict_by_service_uid[service_uid] = r
            except IndexError:
                from bika.lims import logger
                error = "No Analysis Service found for Keyword '%s'. "\
                        "Related: LIMS-1614"
                logger.exception(error, keyword)

        return json.dumps(rr_dict_by_service_uid)

    def get_spec_from_ar(self, ar, keyword):
        empty = {'min': '', 'max': '', 'error': '', 'keyword':keyword}
        spec = ar.getResultsRange()
        if spec:
            return dicts_to_dict(spec, 'keyword').get(keyword, empty)
        return empty

    def isItemAllowed(self, obj):
        """
        It checks if the item can be added to the list depending on the
        department filter. If the analysis service is not assigned to a
        department, show it.
        """
        # Gettin the department from analysis service
        obj_dep = obj.getDepartment()
        result = True
        if obj_dep:
            # Getting the cookie value
            cookie_dep_uid = self.request.get('filter_by_department_info', 'no')
            # Comparing departments' UIDs
            result = True if obj_dep.UID() in\
                self.request.get('filter_by_department_info', 'no') else False
            return result
        return result

    def folderitems(self):
        self.categories = []

        analyses = self.context.getAnalyses(full_objects=True)
        self.analyses = dict([(a.getServiceUID(), a) for a in analyses])
        self.selected = self.analyses.keys()
        self.show_categories = \
            self.context.bika_setup.getCategoriseAnalysisServices()
        self.expand_all_categories = False

        wf = getToolByName(self.context, 'portal_workflow')
        mtool = getToolByName(self.context, 'portal_membership')

        self.allow_edit = mtool.checkPermission('Modify portal content',
                                                self.context)

        items = BikaListingView.folderitems(self)
        analyses = self.context.getAnalyses(full_objects=True)

        parts = self.context.getSample().objectValues('SamplePartition')
        partitions = [{'ResultValue': o.Title(),
                       'ResultText': o.getId()}
                      for o in parts
                      if wf.getInfoFor(o, 'cancellation_state', '') == 'active']
        for x in range(len(items)):
            if not 'obj' in items[x]:
                continue
            obj = items[x]['obj']

            cat = obj.getCategoryTitle()
            items[x]['category'] = cat
            if cat not in self.categories:
                self.categories.append(cat)

            items[x]['selected'] = items[x]['uid'] in self.selected

            items[x]['class']['Title'] = 'service_title'

            # js checks in row_data if an analysis may be removed.
            row_data = {}
            # keyword = obj.getKeyword()
            # if keyword in review_states.keys() \
            #    and review_states[keyword] not in ['sample_due',
            #                                       'to_be_sampled',
            #                                       'to_be_preserved',
            #                                       'sample_received',
            #                                       ]:
            #     row_data['disabled'] = True
            items[x]['row_data'] = json.dumps(row_data)

            calculation = obj.getCalculation()
            items[x]['Calculation'] = calculation and calculation.Title()

            locale = locales.getLocale('en')
            currency = self.context.bika_setup.getCurrency()
            symbol = locale.numbers.currencies[currency].symbol
            items[x]['before']['Price'] = symbol
            items[x]['Price'] = obj.getPrice()
            items[x]['class']['Price'] = 'nowrap'
            items[x]['Priority'] = ''

            if items[x]['selected']:
                items[x]['allow_edit'] = ['Partition', 'min', 'max', 'error']
                if not logged_in_client(self.context):
                    items[x]['allow_edit'].append('Price')

            items[x]['required'].append('Partition')
            items[x]['choices']['Partition'] = partitions

            if obj.UID() in self.analyses:
                analysis = self.analyses[obj.UID()]
                part = analysis.getSamplePartition()
                part = part and part or obj
                items[x]['Partition'] = part.Title()
                spec = self.get_spec_from_ar(self.context,
                                             analysis.getService().getKeyword())
                items[x]["min"] = spec.get("min",'')
                items[x]["max"] = spec.get("max",'')
                items[x]["error"] = spec.get("error",'')
                # Add priority premium
                items[x]['Price'] = analysis.getPrice()
                priority = analysis.getPriority()
                items[x]['Priority'] = priority and priority.Title() or ''
            else:
                items[x]['Partition'] = ''
                items[x]["min"] = ''
                items[x]["max"] = ''
                items[x]["error"] = ''
                items[x]["Priority"] = ''

            after_icons = ''
            if obj.getAccredited():
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/accredited.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Accredited"))
                )
            if obj.getReportDryMatter():
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/dry.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Can be reported as dry matter"))
                )
            if obj.getAttachmentOption() == 'r':
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/attach_reqd.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_("Attachment required"))
                )
            if obj.getAttachmentOption() == 'n':
                after_icons += "<img\
                src='%s/++resource++bika.lims.images/attach_no.png'\
                title='%s'>" % (
                    self.portal_url,
                    t(_('Attachment not permitted'))
                )
            if after_icons:
                items[x]['after']['Title'] = after_icons

            # Display analyses for this Analysis Service in results?
            ser = self.context.getAnalysisServiceSettings(obj.UID())
            items[x]['allow_edit'] = ['Hidden', ]
            items[x]['Hidden'] = ser.get('hidden', obj.getHidden())

        self.categories.sort()
        return items
<|endoftext|>


<commit_before>from sfepy.base.base import *
from sfepy.base.log import Log
from sfepy.solvers.solvers import NonlinearSolver
from sfepy.solvers.nls import Newton, conv_test

class SemismoothNewton(Newton):
    r"""
    The semi-smooth Newton method for solving problems of the following
    structure:

    .. math::
        \begin{split}
          & F(y) = 0 \\
          & A(y) \ge 0 \;,\ B(y) \ge 0 \;,\ \langle A(y), B(y) \rangle = 0
        \end{split}

    The function :math:`F(y)` represents the smooth part of the problem.

    Regular step: :math:`y \leftarrow y - J(y)^{-1} Phi(y)`

    Steepest descent step: :math:`y \leftarrow y - \beta J(y) Phi(y)`

    Notes
    -----
    Although fun_smooth_grad() computes the gradient of the smooth part only,
    it should return the global matrix, where the non-smooth part is
    uninitialized, but pre-allocated.
    """
    name = 'nls.semismooth_newton'

    _colors = {'regular' : 'g', 'steepest_descent' : 'k'}

    def _get_error(self, vec):
        return err

    def __call__(self, vec_x0, conf=None, fun_smooth=None, fun_smooth_grad=None,
                 fun_a=None, fun_a_grad=None, fun_b=None, fun_b_grad=None,
                 lin_solver=None, status=None):

        conf = get_default(conf, self.conf)

        fun_smooth = get_default(fun_smooth, self.fun_smooth)
        fun_smooth_grad = get_default(fun_smooth_grad, self.fun_smooth_grad)
        fun_a = get_default(fun_a, self.fun_a)
        fun_a_grad = get_default(fun_a_grad, self.fun_a_grad)
        fun_b = get_default(fun_b, self.fun_b)
        fun_b_grad = get_default(fun_b_grad, self.fun_b_grad)

        lin_solver = get_default( lin_solver, self.lin_solver )
        status = get_default( status, self.status )

        time_stats = {}

        vec_x_cur = vec_x0.copy()
        vec_x_last = vec_x0.copy()
        vec_dx = None

        if self.log is not None:
            self.log.plot_vlines(color='r', linewidth=1.0)

        err0 = -1.0
        err_last = -1.0
        it = 0
        step_mode = 'regular'
        r_last = None
        while 1:

            ls = 1.0
            vec_dx0 = vec_dx;
            while 1:
                tt = time.clock()

                try:
                    vec_smooth_r = fun_smooth(vec_x)
                    vec_a_r = fun_a(vec_x)
                    vec_b_r = fun_b(vec_x)

                except ValueError, exc:
                    vec_smooth_r = vec_semismooth_r = None
                    ok = False

                else:
                    # Semi-smooth equation.
                    vec_semismooth_r = nm.sqrt(vec_a_r**2.0 + vec_b_r**2.0) \
                                       - (vec_a_r + vec_b_r)
                    r_last = (vec_smooth_r, vec_a_r, vec_b_r, vec_semismooth_r)
                    ok = True

                time_stats['rezidual'] = time.clock() - tt

                if ok:
                    vec_r = nm.r_[vec_smooth_r, vec_semismooth_r]

                    try:
                        err = nla.norm(vec_r)
                    except:
                        output('infs or nans in the residual:', vec_semismooth_r)
                        output(nm.isfinite(vec_semismooth_r).all())
                        debug()

                    if self.log is not None:
                        self.log(err, it)

                    if it == 0:
                        err0 = err;
                        break

                    if err < (err_last * conf.ls_on):
                        step_mode = 'regular'
                        output('regular step')
                        break

                    elif step_mode == 'regular':
                        output('restore previous state')
                        vec_x = vec_x_last.copy()
                        vec_smooth_r, vec_a_r, vec_b_r, vec_semismooth_r = r_last

                        output('steepest descent step')
                        step_mode = 'steepest_descent'
                        break

                    else:
                        output('steepest descent step line search')

                        red = conf.ls_red;
                        output('iter %d, (%.5e < %.5e) (new ls: %e)'\
                               % (it, err, err_last * conf.ls_on, red * ls))

                else: # Failed to compute rezidual.
                    red = conf.ls_red_warp;
                    output('rezidual computation failed for iter %d'
                           ' (new ls: %e)!' % (it, red * ls))
                    if (it == 0):
                        raise RuntimeError('giving up...')

                if ls < conf.ls_min:
                    if not ok:
                        raise RuntimeError('giving up...')
                    output('linesearch failed, continuing anyway')
                    break

                ls *= red;

                vec_dx = ls * vec_dx0;
                vec_x = vec_x_last.copy() - vec_dx

            # End residual loop.

            if self.log is not None:
                self.log.plot_vlines([1],
                                     color=self._colors[step_mode],
                                     linewidth=0.5)

            err_last = err;
            vec_x_last = vec_x.copy()

            condition = conv_test( conf, it, err, err0 )
            if condition >= 0:
                break

            tt = time.clock()

            try:
                mtx_jac = fun_smooth_grad(vec_x)
                mtx_a = fun_a_grad(vec_x)
                mtx_b = fun_b_grad(vec_x)

            except ValueError:
                ok = False

            else:
                ok = True

                n_s = vec_smooth_r.shape[0]
                n_ns = vec_a_r.shape[0]

                iz = nm.where((nm.abs(vec_a_r) < conf.macheps)
                              & (nm.abs(vec_b_r) < conf.macheps))[0]
                inz = nm.setdiff1d(nm.arange(n_ns), iz)

                # Non-active part of the jacobian.
                a_r_nz = vec_a_r[inz]
                b_r_nz = vec_b_r[inz]

                sqrt_ab = nm.sqrt(a_r_nz**2.0 + b_r_nz**2.0)
                mul_a = (a_r_nz / sqrt_ab) - 1.0
                mul_b = (b_r_nz / sqrt_ab) - 1.0

                aux = (mul_a[:,None] * mtx_a[inz]) + (mul_b[:,None] * mtx_b[inz])
                mtx_jac[n_s+inz] = aux

                # Active part of the jacobian.
                vec_z = nm.zeros_like(vec_a_r)
                vec_z[iz] = 1.0

                mtx_a_z = mtx_a[iz]
                mtx_b_z = mtx_b[iz]

                sqrt_ab = nm.sqrt((mtx_a_z.T * mtx_a_z)
                                  + (mtx_b_z.T * mtx_b_z))
                mul_a = mtx_a_z * vec_z
                mul_b = mtx_b_z * vec_z

                aux = (mul_a[:,None] * mtx_a[iz]) + (mul_b[:,None] * mtx_b[iz])
                mtx_jac[n_s+iz] = aux

            time_stats['matrix'] = time.clock() - tt

            if not ok:
                raise RuntimeError('giving up...')

            tt = time.clock()

            if step_mode == 'regular':
                vec_dx = lin_solver(vec_r, mtx=mtx_jac)

            else:
                vec_dx = mtx_jac * vec_r

            time_stats['solve'] = time.clock() - tt

            for kv in time_stats.iteritems():
                output( '%10s: %7.2f [s]' % kv )

            vec_e = mtx_jac * vec_dx - vec_r
            lerr = nla.norm(vec_e)
            if lerr > (conf.eps_a * conf.lin_red):
                output('linear system not solved! (err = %e)' % lerr)

            vec_x -= vec_dx
            it += 1

        if status is not None:
            status['time_stats'] = time_stats
            status['err0'] = err0
            status['err'] = err
            status['condition'] = condition

        if conf.log.plot is not None:
            if self.log is not None:
                self.log(save_figure=conf.log.plot)

        return vec_x
<commit_msg>fix/update semi-smooth Newton solver<commit_after>from sfepy.base.base import *
from sfepy.base.log import Log
from sfepy.solvers.solvers import NonlinearSolver
from sfepy.solvers.nls import Newton, conv_test

class SemismoothNewton(Newton):
    r"""
    The semi-smooth Newton method for solving problems of the following
    structure:

    .. math::
        \begin{split}
          & F(y) = 0 \\
          & A(y) \ge 0 \;,\ B(y) \ge 0 \;,\ \langle A(y), B(y) \rangle = 0
        \end{split}

    The function :math:`F(y)` represents the smooth part of the problem.

    Regular step: :math:`y \leftarrow y - J(y)^{-1} Phi(y)`

    Steepest descent step: :math:`y \leftarrow y - \beta J(y) Phi(y)`

    Notes
    -----
    Although fun_smooth_grad() computes the gradient of the smooth part only,
    it should return the global matrix, where the non-smooth part is
    uninitialized, but pre-allocated.
    """
    name = 'nls.semismooth_newton'

    _colors = {'regular' : 'g', 'steepest_descent' : 'k'}

    def _get_error(self, vec):
        return err

    def __call__(self, vec_x0, conf=None, fun_smooth=None, fun_smooth_grad=None,
                 fun_a=None, fun_a_grad=None, fun_b=None, fun_b_grad=None,
                 lin_solver=None, status=None):

        conf = get_default(conf, self.conf)

        fun_smooth = get_default(fun_smooth, self.fun_smooth)
        fun_smooth_grad = get_default(fun_smooth_grad, self.fun_smooth_grad)
        fun_a = get_default(fun_a, self.fun_a)
        fun_a_grad = get_default(fun_a_grad, self.fun_a_grad)
        fun_b = get_default(fun_b, self.fun_b)
        fun_b_grad = get_default(fun_b_grad, self.fun_b_grad)

        lin_solver = get_default( lin_solver, self.lin_solver )
        status = get_default( status, self.status )

        time_stats = {}

        vec_x = vec_x0.copy()
        vec_x_last = vec_x0.copy()
        vec_dx = None

        if self.log is not None:
            self.log.plot_vlines(color='r', linewidth=1.0)

        err0 = -1.0
        err_last = -1.0
        it = 0
        step_mode = 'regular'
        r_last = None
        reuse_matrix = False
        while 1:

            ls = 1.0
            vec_dx0 = vec_dx;
            i_ls = 0
            while 1:
                tt = time.clock()

                try:
                    vec_smooth_r = fun_smooth(vec_x)
                    vec_a_r = fun_a(vec_x)
                    vec_b_r = fun_b(vec_x)

                except ValueError, exc:
                    vec_smooth_r = vec_semismooth_r = None
                    ok = False

                else:
                    # Semi-smooth equation.
                    vec_semismooth_r = nm.sqrt(vec_a_r**2.0 + vec_b_r**2.0) \
                                       - (vec_a_r + vec_b_r)
                    r_last = (vec_smooth_r, vec_a_r, vec_b_r, vec_semismooth_r)
                    ok = True

                time_stats['rezidual'] = time.clock() - tt

                if ok:
                    vec_r = nm.r_[vec_smooth_r, vec_semismooth_r]

                    try:
                        err = nla.norm(vec_r)
                    except:
                        output('infs or nans in the residual:', vec_semismooth_r)
                        output(nm.isfinite(vec_semismooth_r).all())
                        debug()

                    if self.log is not None:
                        self.log(err, it)

                    if it == 0:
                        err0 = err;
                        break

                    if err < (err_last * conf.ls_on):
                        step_mode = 'regular'
                        break

                    else:
                        output('%s step line search' % step_mode)

                        red = conf.ls_red;
                        output('iter %d, (%.5e < %.5e) (new ls: %e)'\
                               % (it, err, err_last * conf.ls_on, red * ls))

                else: # Failed to compute rezidual.
                    red = conf.ls_red_warp;
                    output('rezidual computation failed for iter %d'
                           ' (new ls: %e)!' % (it, red * ls))
                    if (it == 0):
                        raise RuntimeError('giving up...')

                if ls < conf.ls_min:
                    if not ok:
                        raise RuntimeError('giving up...')

                    if step_mode == 'regular':
                        output('restore previous state')
                        vec_x = vec_x_last.copy()
                        vec_smooth_r, vec_a_r, vec_b_r, vec_semismooth_r = r_last
                        err = err_last
                        reuse_matrix = True

                        step_mode = 'steepest_descent'

                    else:
                        output('linesearch failed, continuing anyway')

                    break

                ls *= red;

                vec_dx = ls * vec_dx0;
                vec_x = vec_x_last.copy() - vec_dx

                i_ls += 1

            # End residual loop.

            output('%s step' % step_mode)

            if self.log is not None:
                self.log.plot_vlines([1],
                                     color=self._colors[step_mode],
                                     linewidth=0.5)

            err_last = err;
            vec_x_last = vec_x.copy()

            condition = conv_test(conf, it, err, err0)
            if condition >= 0:
                break

            tt = time.clock()

            if not reuse_matrix:
                ok, mtx_jac = self.compute_jacobian(vec_x, fun_smooth_grad,
                                                    fun_a_grad, fun_b_grad,
                                                    vec_smooth_r,
                                                    vec_a_r, vec_b_r)

            else:
                ok = True
                reuse_matrix = False

            time_stats['matrix'] = time.clock() - tt

            if not ok:
                raise RuntimeError('giving up...')

            tt = time.clock()

            if step_mode == 'regular':
                vec_dx = lin_solver(vec_r, mtx=mtx_jac)

                vec_e = mtx_jac * vec_dx - vec_r
                lerr = nla.norm(vec_e)
                if lerr > (conf.eps_a * conf.lin_red):
                    output('linear system not solved! (err = %e)' % lerr)

                    output('switching to steepest descent step')
                    step_mode = 'steepest_descent'
                    vec_dx = mtx_jac.T * vec_r

                    debug()

            else:
                vec_dx = mtx_jac.T * vec_r

            time_stats['solve'] = time.clock() - tt

            for kv in time_stats.iteritems():
                output( '%10s: %7.2f [s]' % kv )

            vec_x -= vec_dx
            it += 1

        if status is not None:
            status['time_stats'] = time_stats
            status['err0'] = err0
            status['err'] = err
            status['condition'] = condition

        if conf.log.plot is not None:
            if self.log is not None:
                self.log(save_figure=conf.log.plot)

        return vec_x

    def compute_jacobian(self, vec_x, fun_smooth_grad, fun_a_grad, fun_b_grad,
                         vec_smooth_r, vec_a_r, vec_b_r):
        conf = self.conf

        try:
            mtx_jac = fun_smooth_grad(vec_x)
            mtx_a = fun_a_grad(vec_x)
            mtx_b = fun_b_grad(vec_x)

        except ValueError:
            ok = False

        else:
            ok = True

            n_s = vec_smooth_r.shape[0]
            n_ns = vec_a_r.shape[0]

            aa = nm.abs(vec_a_r)
            ab = nm.abs(vec_b_r)
            iz = nm.where((aa < (conf.macheps * max(aa.max(), 1.0)))
                          & (ab < (conf.macheps * max(ab.max(), 1.0))))[0]
            inz = nm.setdiff1d(nm.arange(n_ns), iz)

            output('non_active/active: %d/%d' % (len(inz), len(iz)))

            mul_a = nm.empty_like(vec_a_r)
            mul_b = nm.empty_like(mul_a)

            # Non-active part of the jacobian.
            if len(inz) > 0:
                a_r_nz = vec_a_r[inz]
                b_r_nz = vec_b_r[inz]

                sqrt_ab = nm.sqrt(a_r_nz**2.0 + b_r_nz**2.0)
                mul_a[inz] = (a_r_nz / sqrt_ab) - 1.0
                mul_b[inz] = (b_r_nz / sqrt_ab) - 1.0

            # Active part of the jacobian.
            if len(iz) > 0:
                vec_z = nm.zeros_like(vec_x)
                vec_z[n_s+iz] = 1.0

                mtx_a_z = mtx_a[iz]
                mtx_b_z = mtx_b[iz]

                sqrt_ab = nm.empty_like(vec_a_r)
                for ir in range(len(iz)):
                    row_a_z = mtx_a_z[ir]
                    row_b_z = mtx_b_z[ir]
                    sqrt_ab[ir] = nm.sqrt((row_a_z * row_a_z.T).todense()
                                          + (row_b_z * row_b_z.T).todense())
                mul_a[iz] = ((mtx_a_z * vec_z) / sqrt_ab) - 1.0
                mul_b[iz] = ((mtx_b_z * vec_z) / sqrt_ab) - 1.0

            # Assume the same sparsity structure!
            for ir in range(n_ns):
                ij0 = mtx_jac.indptr[n_s+ir]
                ij1 = mtx_jac.indptr[n_s+ir+1]
                i0 = mtx_a.indptr[ir]
                i1 = mtx_a.indptr[ir+1]
                assert_((ij1-ij0) == (i1 - i0))
                assert_((mtx_b.indptr[ir+1] - mtx_b.indptr[ir]) == (i1 - i0))

                val = ((mul_a[ir] * mtx_a.data[i0:i1])
                       + (mul_b[ir] * mtx_b.data[i0:i1]))
                mtx_jac.data[ij0:ij1] = val

        return ok, mtx_jac
<|endoftext|>


<commit_before>"""Routines for handling patient lists."""

from __future__ import division, print_function
from functools import total_ordering
from collections import namedtuple
import os.path

import numpy as np

import dwi.asciifile
import dwi.files
import dwi.util

"""Image mode identifier, string representation would be like DWI-Mono-ADCm."""
class ImageMode(object):
    def __init__(self, modality, model='raw', param='raw'):
        self.modality = modality
        self.model = model
        self.param = param

    @staticmethod
    def parse(s):
        lst = s.split('-')
        return ImageMode(*lst)

    def __iter__(self):
        return iter((self.modality, self.model, self.param))

    def __repr__(self):
        return '{}({})'.format(type(self).__name__, ', '.join(iter(self)))

@total_ordering
class GleasonScore(object):
    """Gleason score is a two or three-value measure of prostate cancer
    severity.
    """
    def __init__(self, score):
        """Intialize with a sequence or a string like '3+4+5' (third digit is
        optional)."""
        if isinstance(score, str) or isinstance(score, unicode):
            s = score.split('+')
        else:
            s = score
        s = tuple(int(x) for x in s)
        if len(s) == 2:
            s += (0,) # Internal representation always has three digits.
        if not len(s) == 3:
            raise Exception('Invalid gleason score: %s', score)
        self.score = s

    def __iter__(self):
        score = self.score
        if not score[-1]:
            score = score[0:-1] # Drop trailing zero.
        return iter(score)

    def __repr__(self):
        return '+'.join(str(x) for x in iter(self))

    def __hash__(self):
        return hash(self.score)

    def __iter__(self):
        return iter(self.score)

    def __eq__(self, other):
        return self.score == other.score

    def __lt__(self, other):
        return self.score < other.score

class Lesion(object):
    """Lesion is a lump of cancer tissue."""
    def __init__(self, index, score, location):
        self.index = index # No. in patient.
        self.score = score # Gleason score.
        self.location = location # PZ or CZ.

    def __repr__(self):
        return repr((self.index, self.score, self.location))

@total_ordering
class Patient(object):
    """Patient case."""
    def __init__(self, num, name, scans, lesions):
        self.num = num
        self.name = name
        self.scans = scans
        self.lesions = lesions
        self.score = lesions[0].score # For backwards compatibility.

    def __repr__(self):
        return repr(self.tuple())

    def __hash__(self):
        return hash(self.tuple())

    def __eq__(self, other):
        return self.tuple() == other.tuple()

    def __lt__(self, other):
        return self.tuple() < other.tuple()

    def tuple(self):
        return self.num, self.name, self.scans, self.lesions

def scan_in_patients(patients, num, scan):
    """Is this scan listed in the patients sequence?"""
    for p in patients:
        if p.num == num and scan in p.scans:
            return True
    return False

def get_patient(patients, num):
    """Search a patient from sequence by patient number."""
    for p in patients:
        if p.num == num:
            return p
    raise Exception('Patient not found: {}'.format(num))

def get_gleason_scores(patients):
    """Get all separate Gleason scores, sorted."""
    #return sorted({p.score for p in patients})
    scores = set()
    for p in patients:
        scores.update(l.score for l in p.lesions)
    return sorted(scores)

def score_ord(scores, score):
    """Get Gleason score's ordinal number."""
    return sorted(scores).index(score)

def load_files(patients, filenames, pairs=False):
    """Load pmap files."""
    pmapfiles = []
    if len(filenames) == 1:
        # Workaround for platforms without shell-level globbing.
        import glob
        l = glob.glob(filenames[0])
        if len(l) > 0:
            filenames = l
    for f in filenames:
        num, scan = dwi.util.parse_num_scan(os.path.basename(f))
        if patients is None or scan_in_patients(patients, num, scan):
            pmapfiles.append(f)
    afs = [dwi.asciifile.AsciiFile(x) for x in pmapfiles]
    if pairs:
        dwi.util.scan_pairs(afs)
    ids = [dwi.util.parse_num_scan(af.basename) for af in afs]
    pmaps = [af.a for af in afs]
    pmaps = np.array(pmaps)
    params = afs[0].params()
    assert pmaps.shape[-1] == len(params), 'Parameter name mismatch.'
    #print('Filenames: %i, loaded: %i, lines: %i, columns: %i'
    #        % (len(filenames), pmaps.shape[0], pmaps.shape[1], pmaps.shape[2]))
    return pmaps, ids, params

def load_labels(patients, nums, labeltype='score'):
    """Load labels according to patient numbers."""
    gs = get_gleason_scores(patients)
    scores = [get_patient(patients, n).score for n in nums]
    if labeltype == 'score':
        # Use Gleason score.
        labels = scores
    elif labeltype == 'ord':
        # Use ordinal.
        labels = [score_ord(gs, s) for s in scores]
    elif labeltype == 'bin':
        # Is aggressive? (ROI1.)
        labels = [s > GleasonScore('3+4') for s in scores]
    elif labeltype == 'cancer':
        # Is cancer? (ROI1 vs 2, all true for ROI1.)
        labels = [1] * len(scores)
    else:
        raise Exception('Invalid parameter: %s' % labeltype)
    return labels

def cases_scans(patients, cases=None, scans=None):
    """Generate all case, scan combinations, with optional whitelists."""
    for p in patients:
        if cases is None or p.num in cases:
            for s in p.scans:
                if scans is None or s in scans:
                    yield p.num, s

def lesions(patients):
    """Generate all case, scan, lesion combinations."""
    for p in patients:
        for s in p.scans:
            for l in p.lesions:
                yield p, s, l

# Low group: 3 only; intermediate: 4 secondary or tertiary w/o 5; high: rest.
THRESHOLDS_STANDARD = ('3+3', '3+4')

def read_pmaps(patients_file, pmapdir, thresholds=('3+3',), voxel='all',
               multiroi=False, dropok=False):
    """Read pmaps labeled by their Gleason score.

    Label thresholds are maximum scores of each label group. Labels are ordinal
    of score if no thresholds provided."""
    # TODO Support for selecting measurements over scan pairs
    thresholds = tuple(GleasonScore(x) for x in thresholds)
    patients = dwi.files.read_patients_file(patients_file)
    gs = get_gleason_scores(patients)
    data = []
    for patient, scan, lesion in lesions(patients):
        if not multiroi and lesion.index != 0:
            continue
        case = patient.num
        score = lesion.score
        if thresholds:
            label = sum(score > t for t in thresholds)
        else:
            label = score_ord(gs, score)
        roi = lesion.index if multiroi else None
        try:
            pmap, params, pathname = read_pmap(pmapdir, case, scan, roi=roi,
                                               voxel=voxel)
        except IOError:
            if dropok:
                print('Cannot read pmap for {}, dropping...'.format(
                    (case, scan, roi)))
                continue
            else:
                raise
        d = dict(case=case, scan=scan, roi=lesion.index, score=score,
                 label=label, pmap=pmap, params=params, pathname=pathname)
        data.append(d)
        if pmap.shape != data[0]['pmap'].shape:
            raise Exception('Irregular shape: %s' % pathname)
        if params != data[0]['params']:
            raise Exception('Irregular params: %s' % pathname)
    return data

def grouping(data):
    """Return different scores, grouped scores, and their sample sizes.

    See read_pmaps()."""
    scores = [d['score'] for d in data]
    labels = [d['label'] for d in data]
    n_labels = max(labels) + 1
    groups = [[] for _ in range(n_labels)]
    for s, l in zip(scores, labels):
        groups[l].append(s)
    different_scores = sorted(set(scores))
    group_scores = [sorted(set(g)) for g in groups]
    group_sizes = [len(g) for g in groups]
    return different_scores, group_scores, group_sizes

def read_pmap(dirname, case, scan, roi=None, voxel='all'):
    """Read single pmap."""
    d = dict(d=dirname, c=case, s=scan, r=roi)
    if roi is None:
        s = '{d}/{c}_*{s}*.txt'
    else:
        d['r'] += 1
        s = '{d}/{c}_*{s}_{r}*.txt'
    path = dwi.util.sglob(s.format(**d))
    af = dwi.asciifile.AsciiFile(path)
    pmap = af.a
    params = af.params()
    if pmap.shape[-1] != len(params):
        # TODO Move to Asciifile initializer?
        raise Exception('Number of parameters mismatch: %s' % af.filename)
    # Select voxel to use.
    if voxel == 'all':
        pass # Use all voxels.
    elif voxel == 'sole':
        # Use sole voxel (raise exception if more found).
        if len(pmap) != 1:
            raise Exception('Too many voxels: {}'.format(len(pmap)))
    elif voxel == 'mean':
        pmap = np.mean(pmap, axis=0, keepdims=True) # Use mean voxel.
    elif voxel == 'median':
        pmap = dwi.util.median(pmap, axis=0, keepdims=True) # Use median voxel.
    else:
        pmap = pmap[[int(voxel)]] # Use single voxel only.
    return pmap, params, af.filename
<commit_msg>removed __iter__ redef<commit_after>"""Routines for handling patient lists."""

from __future__ import division, print_function
from functools import total_ordering
import os.path

import numpy as np

import dwi.asciifile
import dwi.files
import dwi.util

"""Image mode identifier, string representation would be like DWI-Mono-ADCm."""
class ImageMode(object):
    def __init__(self, modality, model='raw', param='raw'):
        self.modality = modality
        self.model = model
        self.param = param

    @staticmethod
    def parse(s):
        lst = s.split('-')
        return ImageMode(*lst)

    def __iter__(self):
        return iter((self.modality, self.model, self.param))

    def __repr__(self):
        return '{}({})'.format(type(self).__name__, ', '.join(iter(self)))

@total_ordering
class GleasonScore(object):
    """Gleason score is a two or three-value measure of prostate cancer
    severity.
    """
    def __init__(self, score):
        """Intialize with a sequence or a string like '3+4+5' (third digit is
        optional)."""
        if isinstance(score, str) or isinstance(score, unicode):
            s = score.split('+')
        else:
            s = score
        s = tuple(int(x) for x in s)
        if len(s) == 2:
            s += (0,) # Internal representation always has three digits.
        if not len(s) == 3:
            raise Exception('Invalid gleason score: %s', score)
        self.score = s

    def __iter__(self):
        score = self.score
        if not score[-1]:
            score = score[0:-1] # Drop trailing zero.
        return iter(score)

    def __repr__(self):
        return '+'.join(str(x) for x in iter(self))

    def __hash__(self):
        return hash(self.score)

    def __eq__(self, other):
        return self.score == other.score

    def __lt__(self, other):
        return self.score < other.score

class Lesion(object):
    """Lesion is a lump of cancer tissue."""
    def __init__(self, index, score, location):
        self.index = index # No. in patient.
        self.score = score # Gleason score.
        self.location = location # PZ or CZ.

    def __repr__(self):
        return repr((self.index, self.score, self.location))

@total_ordering
class Patient(object):
    """Patient case."""
    def __init__(self, num, name, scans, lesions):
        self.num = num
        self.name = name
        self.scans = scans
        self.lesions = lesions
        self.score = lesions[0].score # For backwards compatibility.

    def __repr__(self):
        return repr(self.tuple())

    def __hash__(self):
        return hash(self.tuple())

    def __eq__(self, other):
        return self.tuple() == other.tuple()

    def __lt__(self, other):
        return self.tuple() < other.tuple()

    def tuple(self):
        return self.num, self.name, self.scans, self.lesions

def scan_in_patients(patients, num, scan):
    """Is this scan listed in the patients sequence?"""
    for p in patients:
        if p.num == num and scan in p.scans:
            return True
    return False

def get_patient(patients, num):
    """Search a patient from sequence by patient number."""
    for p in patients:
        if p.num == num:
            return p
    raise Exception('Patient not found: {}'.format(num))

def get_gleason_scores(patients):
    """Get all separate Gleason scores, sorted."""
    #return sorted({p.score for p in patients})
    scores = set()
    for p in patients:
        scores.update(l.score for l in p.lesions)
    return sorted(scores)

def score_ord(scores, score):
    """Get Gleason score's ordinal number."""
    return sorted(scores).index(score)

def load_files(patients, filenames, pairs=False):
    """Load pmap files."""
    pmapfiles = []
    if len(filenames) == 1:
        # Workaround for platforms without shell-level globbing.
        import glob
        l = glob.glob(filenames[0])
        if len(l) > 0:
            filenames = l
    for f in filenames:
        num, scan = dwi.util.parse_num_scan(os.path.basename(f))
        if patients is None or scan_in_patients(patients, num, scan):
            pmapfiles.append(f)
    afs = [dwi.asciifile.AsciiFile(x) for x in pmapfiles]
    if pairs:
        dwi.util.scan_pairs(afs)
    ids = [dwi.util.parse_num_scan(af.basename) for af in afs]
    pmaps = [af.a for af in afs]
    pmaps = np.array(pmaps)
    params = afs[0].params()
    assert pmaps.shape[-1] == len(params), 'Parameter name mismatch.'
    #print('Filenames: %i, loaded: %i, lines: %i, columns: %i'
    #        % (len(filenames), pmaps.shape[0], pmaps.shape[1], pmaps.shape[2]))
    return pmaps, ids, params

def load_labels(patients, nums, labeltype='score'):
    """Load labels according to patient numbers."""
    gs = get_gleason_scores(patients)
    scores = [get_patient(patients, n).score for n in nums]
    if labeltype == 'score':
        # Use Gleason score.
        labels = scores
    elif labeltype == 'ord':
        # Use ordinal.
        labels = [score_ord(gs, s) for s in scores]
    elif labeltype == 'bin':
        # Is aggressive? (ROI1.)
        labels = [s > GleasonScore('3+4') for s in scores]
    elif labeltype == 'cancer':
        # Is cancer? (ROI1 vs 2, all true for ROI1.)
        labels = [1] * len(scores)
    else:
        raise Exception('Invalid parameter: %s' % labeltype)
    return labels

def cases_scans(patients, cases=None, scans=None):
    """Generate all case, scan combinations, with optional whitelists."""
    for p in patients:
        if cases is None or p.num in cases:
            for s in p.scans:
                if scans is None or s in scans:
                    yield p.num, s

def lesions(patients):
    """Generate all case, scan, lesion combinations."""
    for p in patients:
        for s in p.scans:
            for l in p.lesions:
                yield p, s, l

# Low group: 3 only; intermediate: 4 secondary or tertiary w/o 5; high: rest.
THRESHOLDS_STANDARD = ('3+3', '3+4')

def read_pmaps(patients_file, pmapdir, thresholds=('3+3',), voxel='all',
               multiroi=False, dropok=False):
    """Read pmaps labeled by their Gleason score.

    Label thresholds are maximum scores of each label group. Labels are ordinal
    of score if no thresholds provided."""
    # TODO Support for selecting measurements over scan pairs
    thresholds = tuple(GleasonScore(x) for x in thresholds)
    patients = dwi.files.read_patients_file(patients_file)
    gs = get_gleason_scores(patients)
    data = []
    for patient, scan, lesion in lesions(patients):
        if not multiroi and lesion.index != 0:
            continue
        case = patient.num
        score = lesion.score
        if thresholds:
            label = sum(score > t for t in thresholds)
        else:
            label = score_ord(gs, score)
        roi = lesion.index if multiroi else None
        try:
            pmap, params, pathname = read_pmap(pmapdir, case, scan, roi=roi,
                                               voxel=voxel)
        except IOError:
            if dropok:
                print('Cannot read pmap for {}, dropping...'.format(
                    (case, scan, roi)))
                continue
            else:
                raise
        d = dict(case=case, scan=scan, roi=lesion.index, score=score,
                 label=label, pmap=pmap, params=params, pathname=pathname)
        data.append(d)
        if pmap.shape != data[0]['pmap'].shape:
            raise Exception('Irregular shape: %s' % pathname)
        if params != data[0]['params']:
            raise Exception('Irregular params: %s' % pathname)
    return data

def grouping(data):
    """Return different scores, grouped scores, and their sample sizes.

    See read_pmaps()."""
    scores = [d['score'] for d in data]
    labels = [d['label'] for d in data]
    n_labels = max(labels) + 1
    groups = [[] for _ in range(n_labels)]
    for s, l in zip(scores, labels):
        groups[l].append(s)
    different_scores = sorted(set(scores))
    group_scores = [sorted(set(g)) for g in groups]
    group_sizes = [len(g) for g in groups]
    return different_scores, group_scores, group_sizes

def read_pmap(dirname, case, scan, roi=None, voxel='all'):
    """Read single pmap."""
    d = dict(d=dirname, c=case, s=scan, r=roi)
    if roi is None:
        s = '{d}/{c}_*{s}*.txt'
    else:
        d['r'] += 1
        s = '{d}/{c}_*{s}_{r}*.txt'
    path = dwi.util.sglob(s.format(**d))
    af = dwi.asciifile.AsciiFile(path)
    pmap = af.a
    params = af.params()
    if pmap.shape[-1] != len(params):
        # TODO Move to Asciifile initializer?
        raise Exception('Number of parameters mismatch: %s' % af.filename)
    # Select voxel to use.
    if voxel == 'all':
        pass # Use all voxels.
    elif voxel == 'sole':
        # Use sole voxel (raise exception if more found).
        if len(pmap) != 1:
            raise Exception('Too many voxels: {}'.format(len(pmap)))
    elif voxel == 'mean':
        pmap = np.mean(pmap, axis=0, keepdims=True) # Use mean voxel.
    elif voxel == 'median':
        pmap = dwi.util.median(pmap, axis=0, keepdims=True) # Use median voxel.
    else:
        pmap = pmap[[int(voxel)]] # Use single voxel only.
    return pmap, params, af.filename
<|endoftext|>


<commit_before>#########################################################################
# Developed in the framework of INDIGO project,
#
# Author: Mario David <mariojmdavid@gmail.com>
#########################################################################

import os
import h5py
import numpy as np
import matplotlib.pyplot as plt

if __name__ == '__main__':

    root_dir = "/home/david/Dropbox/AA-work/bench-run4"
    fout = root_dir + "/bench.hdf"
    haddocks = ['disvis', 'powerfit']
    cases = {'disvis': ['RNA-polymerase-II', 'PRE5-PUP2-complex'],
             'powerfit': ['GroEL-GroES', 'RsgA-ribosome']}

    angles = ['5.0', '10.0']
    voxspac = ['1', '2']
    type = 'GPU'
    nvidia = ['QK5200', 'TK40']

    machs = {'QK5200': ['Phys-C7-QK5200', 'Dock-C7-QK5200', 'Dock-U16-QK5200',
                        'UDock-C7-QK5200', 'UDock-U16-QK5200'],
             'TK40': ['VM-U16-TK40', 'Dock-C7-TK40', 'Dock-U16-TK40',
                      'UDock-C7-TK40', 'UDock-U16-TK40']}

    machshort = {'QK5200': ['Phys-C7', 'Dock-C7', 'Dock-U16', 'UDock-C7', 'UDock-U16'],
                 'TK40': ['VM-U16', 'Dock-C7', 'Dock-U16', 'UDock-C7', 'UDock-U16']}

    lrunsG = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
    initName = 'time-'

    nbars = 5
    index = np.arange(nbars)
    bar_width = 0.35
    error_config = {'ecolor': '#000000', 'lw': 2, 'capsize': 5, 'capthick': 2}

    with h5py.File(fout, 'w') as f:
        for case in cases['disvis']:
            for angle in angles:
                for vs in voxspac:
                    for nv in nvidia:
                        list_mean = []
                        list_stdev = []
                        for mach in machs[nv]:
                            grp_name = 'disvis' + '/' +\
                                       case + '-' + angle + '-' + vs + '/' +\
                                       nv + '/' + mach
                            grp1 = f.create_group(grp_name)
                            npres = np.arange(10)
                            for n in lrunsG:
                                tr_file = root_dir + os.sep + mach + os.sep + \
                                          initName + case + os.sep + 'tr_ang-' + angle + \
                                          '-vs-' + vs + '-type-GPU' + '-n-' + n + '.txt'
                                try:
                                    fraw = open(tr_file)
                                    npres[int(n)-1] = float(fraw.readline())
                                except IOError:
                                    print 'Cannot open', tr_file
                            res = grp1.create_dataset("runtime", data=npres)
                            res.attrs['mean'] = np.mean(npres)
                            res.attrs['stdev'] = np.std(npres)
                            list_mean.append(np.mean(npres))
                            list_stdev.append(np.std(npres))

                        ymin = np.amin(list_mean) - 1.5*np.amax(list_stdev)
                        ymax = np.amax(list_mean) + 1.5*np.amax(list_stdev)
                        fig = plt.figure()
                        plt.bar(index, list_mean, bar_width,
                                color='#ECBDA3',
                                yerr=list_stdev,
                                error_kw=error_config,
                                label='Time (s)')

                        plt.xlabel('Machine')
                        plt.ylabel('Run time (sec)')
                        plt.ylim((ymin, ymax))
                        plt.title('Disvis: case = ' + case + '\n Angle = ' + angle +
                                  ' Voxelspacing = ' + vs + ' GPU = ' + nv)
                        plt.xticks(index, machshort[nv])
                        plt.grid(b=None, which='major', axis='y')
                        plt.legend()
                        plt.savefig(root_dir + '/plots/' + 'disvis ' + case + '-' + angle + '-' + vs + '-' + nv + '.png')

        for case in cases['powerfit']:
            for nv in nvidia:
                list_mean = []
                list_stdev = []
                for mach in machs[nv]:
                    grp_name = 'powerfit' + '/' + case + '/' + nv + '/' + mach
                    grp1 = f.create_group(grp_name)
                    npres = np.arange(10)
                    for n in lrunsG:
                        tr_file = root_dir + os.sep + mach + os.sep + initName + case + os.sep + 'tr_ang-4.71' + \
                                  '-type-GPU' + '-n-' + n + '.txt'
                        try:
                            fraw = open(tr_file)
                            npres[int(n) - 1] = float(fraw.readline())
                        except IOError:
                            print 'Cannot open', tr_file
                    res = grp1.create_dataset("runtime", data=npres)
                    res.attrs['mean'] = np.mean(npres)
                    res.attrs['stdev'] = np.std(npres)
                    list_mean.append(np.mean(npres))
                    list_stdev.append(np.std(npres))

                ymin = np.amin(list_mean) - 1.5*np.amax(list_stdev)
                ymax = np.amax(list_mean) + 1.5*np.amax(list_stdev)
                fig = plt.figure()
                plt.bar(index, list_mean, bar_width,
                        color='#ECBDA3',
                        yerr=list_stdev,
                        error_kw=error_config,
                        label='Time (s)')

                plt.xlabel('Machine')
                plt.ylabel('Run time (sec)')
                plt.ylim((ymin, ymax))
                plt.title('Powerfit: Case = ' + case + '\n GPU = ' + nv)
                plt.xticks(index, machshort[nv])
                plt.grid(b=None, which='major', axis='y')
                plt.legend()
                plt.savefig(root_dir + '/plots/' + 'powerfit ' + case + '-' + nv + '.png')
<commit_msg>add sort of array<commit_after>#########################################################################
# Developed in the framework of INDIGO project,
#
# Author: Mario David <mariojmdavid@gmail.com>
#########################################################################

import os
import h5py
import numpy as np
import matplotlib.pyplot as plt

if __name__ == '__main__':

    root_dir = "/home/david/Dropbox/AA-work/bench-run4"
    fout = root_dir + "/bench.hdf"
    haddocks = ['disvis', 'powerfit']
    cases = {'disvis': ['RNA-polymerase-II', 'PRE5-PUP2-complex'],
             'powerfit': ['GroEL-GroES', 'RsgA-ribosome']}

    angles = ['5.0', '10.0']
    voxspac = ['1', '2']
    type = 'GPU'
    nvidia = ['QK5200', 'TK40']

    machs = {'QK5200': ['Phys-C7-QK5200', 'Dock-C7-QK5200', 'Dock-U16-QK5200',
                        'UDock-C7-QK5200', 'UDock-U16-QK5200'],
             'TK40': ['VM-U16-TK40', 'Dock-C7-TK40', 'Dock-U16-TK40',
                      'UDock-C7-TK40', 'UDock-U16-TK40']}

    machshort = {'QK5200': ['Phys-C7', 'Dock-C7', 'Dock-U16', 'UDock-C7', 'UDock-U16'],
                 'TK40': ['VM-U16', 'Dock-C7', 'Dock-U16', 'UDock-C7', 'UDock-U16']}

    lrunsG = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
    initName = 'time-'

    nbars = 5
    index = np.arange(nbars)
    bar_width = 0.35
    error_config = {'ecolor': '#000000', 'lw': 2, 'capsize': 5, 'capthick': 2}

    with h5py.File(fout, 'w') as f:
        for case in cases['disvis']:
            for angle in angles:
                for vs in voxspac:
                    for nv in nvidia:
                        list_mean = []
                        list_stdev = []
                        for mach in machs[nv]:
                            grp_name = 'disvis' + '/' +\
                                       case + '-' + angle + '-' + vs + '/' +\
                                       nv + '/' + mach
                            grp1 = f.create_group(grp_name)
                            npres = np.arange(10)
                            for n in lrunsG:
                                tr_file = root_dir + os.sep + mach + os.sep + \
                                          initName + case + os.sep + 'tr_ang-' + angle + \
                                          '-vs-' + vs + '-type-GPU' + '-n-' + n + '.txt'
                                try:
                                    fraw = open(tr_file)
                                    npres[int(n)-1] = float(fraw.readline())
                                except IOError:
                                    print 'Cannot open', tr_file
                            npres.sort()
                            res = grp1.create_dataset("runtime", data=npres)
                            res.attrs['mean'] = np.mean(npres)
                            res.attrs['stdev'] = np.std(npres)
                            list_mean.append(np.mean(npres))
                            list_stdev.append(np.std(npres))

                        ymin = np.amin(list_mean) - 1.5*np.amax(list_stdev)
                        ymax = np.amax(list_mean) + 1.5*np.amax(list_stdev)
                        fig = plt.figure()
                        plt.bar(index, list_mean, bar_width,
                                color='#ECBDA3',
                                yerr=list_stdev,
                                error_kw=error_config,
                                label='Time (s)')

                        plt.xlabel('Machine')
                        plt.ylabel('Run time (sec)')
                        plt.ylim((ymin, ymax))
                        plt.title('Disvis: case = ' + case + '\n Angle = ' + angle +
                                  ' Voxelspacing = ' + vs + ' GPU = ' + nv)
                        plt.xticks(index, machshort[nv])
                        plt.grid(b=None, which='major', axis='y')
                        plt.legend()
                        plt.savefig(root_dir + '/plots/' + 'disvis ' + case + '-' + angle + '-' + vs + '-' + nv + '.png')

        for case in cases['powerfit']:
            for nv in nvidia:
                list_mean = []
                list_stdev = []
                for mach in machs[nv]:
                    grp_name = 'powerfit' + '/' + case + '/' + nv + '/' + mach
                    grp1 = f.create_group(grp_name)
                    npres = np.arange(10)
                    for n in lrunsG:
                        tr_file = root_dir + os.sep + mach + os.sep + initName + case + os.sep + 'tr_ang-4.71' + \
                                  '-type-GPU' + '-n-' + n + '.txt'
                        try:
                            fraw = open(tr_file)
                            npres[int(n) - 1] = float(fraw.readline())
                        except IOError:
                            print 'Cannot open', tr_file
                    npres.sort()
                    res = grp1.create_dataset("runtime", data=npres)
                    res.attrs['mean'] = np.mean(npres)
                    res.attrs['stdev'] = np.std(npres)
                    list_mean.append(np.mean(npres))
                    list_stdev.append(np.std(npres))

                ymin = np.amin(list_mean) - 1.5*np.amax(list_stdev)
                ymax = np.amax(list_mean) + 1.5*np.amax(list_stdev)
                fig = plt.figure()
                plt.bar(index, list_mean, bar_width,
                        color='#ECBDA3',
                        yerr=list_stdev,
                        error_kw=error_config,
                        label='Time (s)')

                plt.xlabel('Machine')
                plt.ylabel('Run time (sec)')
                plt.ylim((ymin, ymax))
                plt.title('Powerfit: Case = ' + case + '\n GPU = ' + nv)
                plt.xticks(index, machshort[nv])
                plt.grid(b=None, which='major', axis='y')
                plt.legend()
                plt.savefig(root_dir + '/plots/' + 'powerfit ' + case + '-' + nv + '.png')
<|endoftext|>


<commit_before>import gdb

class StaticVectorPrinter(object):
    """Pretty-printer for static_vectors."""

    def __init__(self, val):
        self.val = val
        self.type = val.type
        self.itype = val.type.template_argument(0)

    def display_hint(self):
        return 'array'

    def to_string(self):
        return "%s of length %d" % (self.type, self.val["size_"])

    def children(self):
        items = self.val["data_"].cast(self.itype.pointer())
        for i in xrange(self.val["size_"]):
            yield "[%d]" % i, items.dereference()
            items += 1

def build_pretty_printer():
    pp = gdb.printing.RegexpCollectionPrettyPrinter("xv6")
    pp.add_printer('static_vector', '^static_vector<.*>$', StaticVectorPrinter)
    return pp

gdb.printing.register_pretty_printer(
    gdb.current_objfile(),
    build_pretty_printer())
<commit_msg>import gdb.printing to avoid gdb python error<commit_after>import gdb
import gdb.printing

class StaticVectorPrinter(object):
    """Pretty-printer for static_vectors."""

    def __init__(self, val):
        self.val = val
        self.type = val.type
        self.itype = val.type.template_argument(0)

    def display_hint(self):
        return 'array'

    def to_string(self):
        return "%s of length %d" % (self.type, self.val["size_"])

    def children(self):
        items = self.val["data_"].cast(self.itype.pointer())
        for i in xrange(self.val["size_"]):
            yield "[%d]" % i, items.dereference()
            items += 1

def build_pretty_printer():
    pp = gdb.printing.RegexpCollectionPrettyPrinter("xv6")
    pp.add_printer('static_vector', '^static_vector<.*>$', StaticVectorPrinter)
    return pp

gdb.printing.register_pretty_printer(
    gdb.current_objfile(),
    build_pretty_printer())
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import glob

from optparse import OptionParser

parser = OptionParser()
parser.add_option("-q", "--quiet",
                  action="store_false", dest="verbose", default=True,
                  help="don't print status messages to stdout")

parser.add_option("-C", "--content",
                  action="store_true", dest="docontent", default=False,
                  help="Disect content, rather than the file.")

(options, args) = parser.parse_args()

import content_index, std_index

from content_index import cntIndex

for fname in args:
    try:
        if options.docontent:
            fp = open(fname,'rb')
            content = fp.read()
            fp.close()
            res = cntIndex.doIndex(content, fname, None, None, True)
        else:
            res = cntIndex.doIndex(None, fname, None, fname,True)

    except Exception,e:
        import traceback,sys
        tb_s = reduce(lambda x, y: x+y, traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))

#eof
<commit_msg>Doc index: Small improvement at the helper script.<commit_after>#!/usr/bin/python

import sys
import os
import glob

from optparse import OptionParser

parser = OptionParser()
parser.add_option("-q", "--quiet",
                  action="store_false", dest="verbose", default=True,
                  help="don't print status messages to stdout")

parser.add_option("-C", "--content",
                  action="store_true", dest="docontent", default=False,
                  help="Disect content, rather than the file.")

(options, args) = parser.parse_args()

import content_index, std_index

from content_index import cntIndex

for fname in args:
    try:
        if options.docontent:
            fp = open(fname,'rb')
            content = fp.read()
            fp.close()
            res = cntIndex.doIndex(content, fname, None, None, True)
        else:
            res = cntIndex.doIndex(None, fname, None, fname,True)

        if options.verbose:
            for line in res[:5]:
                print line
    except Exception,e:
        import traceback,sys
        tb_s = reduce(lambda x, y: x+y, traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))

#eof
<|endoftext|>


<commit_before>import json
import os
import re
import shutil

from PblCommand import PblCommand
from PblProjectCreator import *

def read_c_code(c_file_path):

    C_SINGLELINE_COMMENT_PATTERN = '//.*'
    C_MULTILINE_COMMENT_PATTERN = '/\*.*\*/'

    with open(c_file_path, 'r') as f:
        c_code = f.read()

        c_code = re.sub(C_SINGLELINE_COMMENT_PATTERN, '', c_code)
        c_code = re.sub(C_MULTILINE_COMMENT_PATTERN, '', c_code)

        return c_code

def convert_c_uuid(c_uuid):

    C_UUID_BYTE_PATTERN = '0x([0-9A-Fa-f]{2})'
    C_UUID_PATTERN = '^{\s*' + '\s*,\s*'.join([C_UUID_BYTE_PATTERN] * 16) + '\s*}$'

    UUID_FORMAT = "{}{}{}{}-{}{}-{}{}-{}{}-{}{}{}{}{}{}"

    c_uuid = c_uuid.lower()
    if re.match(C_UUID_PATTERN, c_uuid):
        return UUID_FORMAT.format(*re.findall(C_UUID_BYTE_PATTERN, c_uuid))
    else:
        return c_uuid

def extract_c_macros_from_code(c_code, macros={}):

    C_IDENTIFIER_PATTERN = '[A-Za-z_]\w*'
    C_DEFINE_PATTERN = '#define\s+('+C_IDENTIFIER_PATTERN+')\s+\(*(.+)\)*\s*'

    for m in re.finditer(C_DEFINE_PATTERN, c_code):
        groups = m.groups()
        macros[groups[0]] = groups[1].strip()

def extract_c_macros_from_project(project_root, macros={}):
    src_path = os.path.join(project_root, 'src')
    for root, dirnames, filenames in os.walk(src_path):
        for f in filenames:
            file_path = os.path.join(root, f)
            extract_c_macros_from_code(read_c_code(file_path), macros)

    return macros

def convert_c_expr_dict(c_expr_dict, project_root):

    C_STRING_PATTERN = '^"(.*)"$'

    macros = extract_c_macros_from_project(project_root)
    for k, v in c_expr_dict.iteritems():
        if v == None:
            continue

        # Expand C macros
        if v in macros:
            v = macros[v]

        # Format C strings
        m = re.match(C_STRING_PATTERN, v)
        if m:
            v = m.groups()[0].decode('string-escape')

        c_expr_dict[k] = v

    return c_expr_dict

def find_pbl_app_info(project_root):

    C_LITERAL_PATTERN = '([^,]+|"[^"]*")'

    PBL_APP_INFO_PATTERN = (
            'PBL_APP_INFO(?:_SIMPLE)?\(\s*' +
            '\s*,\s*'.join([C_LITERAL_PATTERN] * 4) +
            '(?:\s*,\s*' + '\s*,\s*'.join([C_LITERAL_PATTERN] * 3) + ')?' +
            '\s*\)'
            )

    PBL_APP_INFO_FIELDS = [
            'uuid',
            'name',
            'company_name',
            'version_major',
            'version_minor',
            'menu_icon',
            'type'
            ]

    src_path = os.path.join(project_root, 'src')
    for root, dirnames, filenames in os.walk(src_path):
        for f in filenames:
            file_path = os.path.join(root, f)
            m = re.search(PBL_APP_INFO_PATTERN, read_c_code(file_path))
            if m:
                return dict(zip(PBL_APP_INFO_FIELDS, m.groups()))

def extract_c_appinfo(project_root):

    appinfo_c_def = find_pbl_app_info(project_root)
    if not appinfo_c_def:
        raise Exception("Could not find usage of PBL_APP_INFO")

    appinfo_c_def = convert_c_expr_dict(appinfo_c_def, project_root)

    version_major = int(appinfo_c_def['version_major'] or '1', 0)
    version_minor = int(appinfo_c_def['version_minor'] or '0', 0)

    appinfo_json_def = {
        'uuid': convert_c_uuid(appinfo_c_def['uuid']),
        'project_name': appinfo_c_def['name'],
        'company_name': appinfo_c_def['company_name'],
        'version_code': version_major,
        'version_label': '{}.{}.0'.format(version_major, version_minor),
        'menu_icon': appinfo_c_def['menu_icon'],
        'is_watchface': 'true' if appinfo_c_def['type'] == 'APP_INFO_WATCH_FACE' else 'false',
        'app_keys': '{}',
        'resources_media': '[]',
    }

    return appinfo_json_def

def load_app_keys(js_appinfo_path):
    with open(js_appinfo_path, "r") as f:
        try:
            app_keys = json.load(f)['app_keys']
        except:
            raise Exception("Failed to import app_keys from {} into new appinfo.json".format(js_appinfo_path))

        app_keys = json.dumps(app_keys, indent=2)
        return re.sub('\s*\n', '\n  ', app_keys)

def load_resources_map(resources_map_path, menu_icon_name=None):

    C_RESOURCE_PREFIX = 'RESOURCE_ID_'

    def convert_resources_media_item(item):
        if item['file'] == 'resource_map.json':
            return None
        else:
            item_name = item['defName']
            del item['defName']
            item['name'] = item_name

            if menu_icon_name and C_RESOURCE_PREFIX + item_name == menu_icon_name:
                item['menuIcon'] = True

            return item

    with open(resources_map_path, "r") as f:
        try:
            resources_media = json.load(f)['media']
        except:
            raise Exception("Failed to import {} into appinfo.json".format(resources_map_path))

        resources_media = filter(None, [convert_resources_media_item(item) for item in resources_media])
        resources_media = json.dumps(resources_media, indent=2)
        return re.sub('\s*\n', '\n    ', resources_media)

def generate_appinfo_from_old_project(project_root, js_appinfo_path=None, resources_media_path=None):
    appinfo_json_def = extract_c_appinfo(project_root)

    if js_appinfo_path and os.path.exists(js_appinfo_path):
        appinfo_json_def['app_keys'] = load_app_keys(js_appinfo_path)

    if resources_media_path and os.path.exists(resources_media_path):
        menu_icon_name = appinfo_json_def['menu_icon']
        appinfo_json_def['resources_media'] = load_resources_map(resources_media_path, menu_icon_name)

    with open(os.path.join(project_root, "appinfo.json"), "w") as f:
        f.write(FILE_DUMMY_APPINFO.substitute(**appinfo_json_def))

def convert_project():
    project_root = os.getcwd()

    js_appinfo_path = os.path.join(project_root, 'src/js/appinfo.json')

    resources_path = 'resources/src'
    resources_media_path = os.path.join(project_root, os.path.join(resources_path, 'resource_map.json'))

    generate_appinfo_from_old_project(
            project_root,
            js_appinfo_path=js_appinfo_path,
            resources_media_path=resources_media_path)

    links_to_remove = [
            'include',
            'lib',
            'pebble_app.ld',
            'tools',
            'waf',
            'wscript'
            ]

    for l in links_to_remove:
        if os.path.islink(l):
            os.unlink(l)

    if os.path.exists('.gitignore'):
        os.remove('.gitignore')

    if os.path.exists('.hgignore'):
        os.remove('.hgignore')

    with open("wscript", "w") as f:
        f.write(FILE_WSCRIPT)

    with open(".gitignore", "w") as f:
        f.write(FILE_GITIGNORE)

    if os.path.exists(js_appinfo_path):
        os.remove(js_appinfo_path)

    if os.path.exists(resources_media_path):
        os.remove(resources_media_path)

    if os.path.exists(resources_path):
        try:
            for f in os.listdir(resources_path):
                shutil.move(os.path.join(resources_path, f), os.path.join('resources', f))
            os.rmdir(resources_path)
        except:
            raise Exception("Could not move all files in {} up one level".format(resources_path))

class PblProjectConverter(PblCommand):
    name = 'convert-project'
    help = """convert an existing Pebble project to the current SDK.

Note: This will only convert the project, you'll still have to update your source to match the new APIs."""

    def run(self, args):
        try:
            check_project_directory()
            print "No conversion required"
            return 0
        except OutdatedProjectException:
            convert_project()
            print "Project successfully converted!"
            return 0

<commit_msg>Remove 1.x crap. Add targetPlatforms to appinfo. Call pebble clean.<commit_after>import json
import os
import re
import shutil

from PblCommand import PblCommand
from PblProjectCreator import *

def generate_appinfo_from_old_project(project_root):
    app_info_path = os.path.join(project_root, "appinfo.json")
    with open(app_info_path, "r") as f:
        app_info_json = json.load(f)

    app_info_json["targetPlatforms"] = ["aplite", "basalt"]

    with open(app_info_path, "w") as f:
        json.dump(app_info_json, f, indent=True)

def convert_project():
    project_root = os.getcwd()

    generate_appinfo_from_old_project(project_root)

    wscript_path = os.path.join(project_root, "wscript")

    os.remove(wscript_path)

    with open(wscript_path, "w") as f:
        f.write(FILE_WSCRIPT)

    os.system('pebble clean')

class PblProjectConverter(PblCommand):
    name = 'convert-project'
    help = """convert an existing Pebble project to the current SDK.

Note: This will only convert the project, you'll still have to update your source to match the new APIs."""

    def run(self, args):
        try:
            check_project_directory()
            print "No conversion required"
            return 0
        except OutdatedProjectException:
            convert_project()
            print "Project successfully converted!"
            return 0

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__author__ = 'macpro'

from django.forms import ModelForm
from models import Comments

class CommentForm(ModelForm):
    class Meta:
        model = Comments
        fields = ['comments_text']
<commit_msg>Update forms.py<commit_after># -*- coding: utf-8 -*-
__author__ = 'rom'

from django.forms import ModelForm
from models import Comments

class CommentForm(ModelForm):
    class Meta:
        model = Comments
        fields = ['comments_text']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Scalepack2mtz.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# A wrapper for the jiffy program scalepack2mtz to convert merged
# scalepack files to MTZ format - with an added spacegroup and
# pname/xname/dname stuff.
#

import os
import sys

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                             'Python'))

from Driver.DriverFactory import DriverFactory
from Decorators.DecoratorFactory import DecoratorFactory

def Scalepack2mtz(DriverType = None):
    '''A factory for Scalepack2mtzWrapper classes.'''

    DriverInstance = DriverFactory.Driver(DriverType)
    CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')

    class Scalepack2mtzWrapper(CCP4DriverInstance.__class__):
        '''A wrapper for Scalepack2mtz, using the CCP4-ified Driver.'''

        def __init__(self):
            # generic things
            CCP4DriverInstance.__class__.__init__(self)
            self.set_executable('scalepack2mtz')

            # specific information
            self._pname = None
            self._xname = None
            self._dname = None
            self._spacegroup = None

            return

        def set_spacegroup(self, spacegroup):
            self._spacegroup = spacegroup

        def set_project_info(self, pname, xname, dname):
            self._pname = pname
            self._dname = xname
            self._dname = dname

        def convert(self):
            self.check_hklin()
            self.check_hklout()

            if not self._spacegroup:
                raise RuntimeError, 'spacegroup must be assigned'

            self.start()

            self.input('symmetry %s' % self._spacegroup)
            if self._pname and self._xname and self._dname:
                self.input('name project %s crystal %s dataset %s' % \
                           (self._pname, self._xname, self._dname))

            self.close_wait()

            try:
                self.check_for_errors()
                self.check_ccp4_errors()

            except RuntimeError, e:
                try:
                    os.remove(self.get_hklout())
                except:
                    pass

                raise e

            return

    return Scalepack2mtzWrapper()

if __name__ == '__main__':

    # then run a test

    hklin = os.path.join(os.environ['X2TD_ROOT'],
                         'Test', 'UnitTest', 'Interfaces',
                         'Scaler', 'Merged', 'TS00_13185_scaled_INFL.sca')

    hklout = 'TS00_13185_INFL.mtz'

    s2m = Scalepack2mtz()

    s2m.set_hklin(hklin)
    s2m.set_hklout(hklout)
    s2m.set_spacegroup('P212121')
    s2m.set_project_info('TS00', 'X13185', 'INFL')
    s2m.convert()

<commit_msg>Added set_cell method.<commit_after>#!/usr/bin/env python
# Scalepack2mtz.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# A wrapper for the jiffy program scalepack2mtz to convert merged
# scalepack files to MTZ format - with an added spacegroup and
# pname/xname/dname stuff.
#

import os
import sys

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                             'Python'))

from Driver.DriverFactory import DriverFactory
from Decorators.DecoratorFactory import DecoratorFactory

def Scalepack2mtz(DriverType = None):
    '''A factory for Scalepack2mtzWrapper classes.'''

    DriverInstance = DriverFactory.Driver(DriverType)
    CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')

    class Scalepack2mtzWrapper(CCP4DriverInstance.__class__):
        '''A wrapper for Scalepack2mtz, using the CCP4-ified Driver.'''

        def __init__(self):
            # generic things
            CCP4DriverInstance.__class__.__init__(self)
            self.set_executable('scalepack2mtz')

            # specific information
            self._pname = None
            self._xname = None
            self._dname = None
            self._spacegroup = None

            # optional information
            self._cell = None

            return

        def set_spacegroup(self, spacegroup):
            self._spacegroup = spacegroup

        def set_cell(self, cell):
            self._cell = cell

        def set_project_info(self, pname, xname, dname):
            self._pname = pname
            self._dname = xname
            self._dname = dname

        def convert(self):
            self.check_hklin()
            self.check_hklout()

            if not self._spacegroup:
                raise RuntimeError, 'spacegroup must be assigned'

            self.start()

            self.input('symmetry %s' % self._spacegroup)
            if self._pname and self._xname and self._dname:
                self.input('name project %s crystal %s dataset %s' % \
                           (self._pname, self._xname, self._dname))
            if self._cell:
                self.input('cell %f %f %f %f %f %f' % self._cell)

            self.close_wait()

            try:
                self.check_for_errors()
                self.check_ccp4_errors()

            except RuntimeError, e:
                try:
                    os.remove(self.get_hklout())
                except:
                    pass

                raise e

            return

    return Scalepack2mtzWrapper()

if __name__ == '__main__':

    # then run a test

    hklin = os.path.join(os.environ['X2TD_ROOT'],
                         'Test', 'UnitTest', 'Interfaces',
                         'Scaler', 'Merged', 'TS00_13185_scaled_INFL.sca')

    hklout = 'TS00_13185_INFL.mtz'

    s2m = Scalepack2mtz()

    s2m.set_hklin(hklin)
    s2m.set_hklout(hklout)
    s2m.set_spacegroup('P212121')
    s2m.set_project_info('TS00', 'X13185', 'INFL')
    s2m.convert()

<|endoftext|>


<commit_before>from django.urls import reverse
from haystack import indexes

from .models import Article

class ArticleIndex(indexes.SearchIndex, indexes.Indexable):
    url = indexes.CharField(indexed=False)
    aid = indexes.CharField(model_attr='aid', indexed=False)
    title = indexes.CharField(model_attr='title')
    description = indexes.CharField(model_attr='description')
    text = indexes.CharField(document=True, model_attr='content')

    page = indexes.IntegerField(model_attr='page__id', indexed=False)
    page_number = indexes.IntegerField(
        model_attr='page__number', indexed=False)
    page_image = indexes.CharField(model_attr='page__image', indexed=False)

    issue = indexes.IntegerField(model_attr='issue__id', indexed=False)
    issue_uid = indexes.CharField(model_attr='issue__uid', indexed=False)
    issue_date = indexes.FacetDateField(model_attr='issue__issue_date')
    issue_year = indexes.FacetIntegerField(
        model_attr='issue__issue_date__year')
    issue_number_of_pages = indexes.IntegerField(
        model_attr='issue__number_of_pages', indexed=False)
    # issue_pdf = indexes.CharField(model_attr='issue__pdf', indexed=False)

    publication = indexes.IntegerField(
        model_attr='page__issue__publication__id', indexed=False)
    publication_abbreviation = indexes.FacetCharField(
        model_attr='page__issue__publication__abbreviation')
    publication_title = indexes.FacetCharField(
        model_attr='page__issue__publication__title')
    publication_description = indexes.CharField(
        model_attr='page__issue__publication__description')

    def get_model(self):
        return Article

    def index_queryset(self, using=None):
        return self.get_model().objects.all()

    def prepare_url(self, obj):
        return reverse('article-detail', args=[obj.id])
<commit_msg>Added slug field and issue pdf to the index.<commit_after>from django.urls import reverse
from haystack import indexes

from .models import Article

class ArticleIndex(indexes.SearchIndex, indexes.Indexable):
    url = indexes.CharField(indexed=False)
    aid = indexes.CharField(model_attr='aid', indexed=False)
    title = indexes.CharField(model_attr='title')
    description = indexes.CharField(model_attr='description')
    text = indexes.CharField(document=True, model_attr='content')

    page = indexes.IntegerField(model_attr='page__id', indexed=False)
    page_number = indexes.IntegerField(
        model_attr='page__number', indexed=False)
    page_image = indexes.CharField(model_attr='page__image', indexed=False)

    issue = indexes.IntegerField(model_attr='issue__id', indexed=False)
    issue_uid = indexes.CharField(model_attr='issue__uid', indexed=False)
    issue_slug = indexes.CharField(model_attr='issue__slug', indexed=False)
    issue_date = indexes.FacetDateField(model_attr='issue__issue_date')
    issue_year = indexes.FacetIntegerField(
        model_attr='issue__issue_date__year')
    issue_number_of_pages = indexes.IntegerField(
        model_attr='issue__number_of_pages', indexed=False)
    issue_pdf = indexes.CharField(model_attr='issue__pdf', indexed=False)

    publication = indexes.IntegerField(
        model_attr='page__issue__publication__id', indexed=False)
    publication_abbreviation = indexes.FacetCharField(
        model_attr='page__issue__publication__abbreviation')
    publication_slug = indexes.CharField(
        model_attr='page__issue__publication__slug', indexed=False)
    publication_title = indexes.FacetCharField(
        model_attr='page__issue__publication__title')
    publication_description = indexes.CharField(
        model_attr='page__issue__publication__description')

    def get_model(self):
        return Article

    def index_queryset(self, using=None):
        return self.get_model().objects.all()

    def prepare_url(self, obj):
        return reverse('article-detail', args=[obj.id])
<|endoftext|>