<commit_before># -*- coding: utf-8 -*-
""" override task """

import pprint
import logging
from dateutil import parser
from datetime import datetime, date, timedelta

from openerp import models, fields, api, exceptions, tools
_logger = logging.getLogger(__name__)

class Task(models.Model):
    """ override task """

    _inherit = 'project.task'
    _stage_id_fatturazione = 5002
    _stage_id_specifica = 2
    _stage_id_annullo = 8
    _stage_id_done = 7

    @api.model
    def _needaction_domain_get(self):
        """ _needaction_domain_get remove unused counter on menu """
        return []

    @api.depends('sale_order_state', 'invoiced', 'stage_id')
    def compute_can_be_invoiced(self):
        """ compute_can_be_invoiced """
        for t in self:
            t.can_be_invoiced = (t.sale_order_state == 'manual' and
                                 (t.invoiced == True or not t.stage_id or t.stage_id.id not in [self._stage_id_fatturazione]))

    @api.depends('milestone', 'project_id', 'project_ref_id')
    def compute_billing_project(self):
        """ ricava il progetto di riferimento per la fatturazione """
        for task in self:
            if task.milestone:
                task.billing_project = task.project_id
            else:
                task.billing_project = task.project_ref_id

    def search_billing_project(self, operator, value):
        """ search_billing_project """
        if operator == 'like':
            operator = 'ilike'
        return [
            '|',
            ('project_ref_id.name', operator, value),
            ('project_id.name', operator, value)
            ]

    @api.depends('planned_hours', 'user_id')
    def compute_cost(self):
        """ ricava il costo orario dal contratto  """
        for task in self:
            resource = self.env['resource.resource'].sudo().search(
                [
                    ('user_id', '=', task.user_id.id)
                ]
            )
            employee = self.env['hr.employee'].sudo().search(
                [
                    ('resource_id', '=', resource.id)
                ]
            )
            cost = 0

            if resource and employee and employee.contract_id:
                hour_cost = employee.contract_id.wage
                cost = hour_cost * task.planned_hours

            task.cost = cost

    @api.depends('direct_sale_line_id', 'direct_sale_line_id.price_unit', 'cost', 'points')
    def compute_price(self):
        """ ricalcola il prezzo basato sui punti o sulla riga di offerta  """
        for task in self.sudo():
            if task.points:
                """
                ## based on points
                """
                task.price = task.an_acc_by_prj.point_unit_price * task.points
            else:
                """
                    ## base on sale offer
                """
                row_price = 0
                price = 0

                if(task.direct_sale_line_id and
                   isinstance(task.direct_sale_line_id.id, (int, long))):

                    line = task.direct_sale_line_id

                    row_price = line.price_unit * line.product_uom_qty
                    self._cr.execute("select sum(cost), case when count(id) > 0 then count(id) else 1 end  from project_task "
                                     " where direct_sale_line_id = %s and stage_id <> %s" %
                                     (line.id, self._stage_id_annullo))
                    record = self._cr.fetchone()

                    _logger.info(pprint.pformat(record))

                    """ costo calcolato in proporzione al prezzo dei task
                    o in proporzione al loro numero se il prezzo totale è 0 """
                    cost_factor = task.cost/float(record[0]) if record[0] else 1/float(record[1])

                    _logger.info("recalc cost/price %s" % cost_factor)

                    price = cost_factor * row_price

                task.price = price

    def _default_invoice_date_cache(self):
        if 'budget_line_cache_invoice_date' in self.cache:
            return self.cache['budget_line_cache_invoice_date']
        return False

    def _default_planned_amount_cache(self):
        if 'budget_line_cache_planned_amount' in self.cache:
            return self.cache['budget_line_cache_planned_amount']
        return False

    """
        ####
        ####  FIELDS
        ####
    """
    points = fields.Integer(string='Points')
    project_ref_id = fields.Many2one('project.project', 'Project reference')
    an_acc_by_prj = fields.Many2one('account.analytic.account',
                                    string="Contract/Analytic",
                                    related='project_id.analytic_account_id',
                                    readonly=True
                                   )
    an_acc_by_prj_ref = fields.Many2one('account.analytic.account',
                                        string="Contract/Analytic reference",
                                        related='project_ref_id.analytic_account_id',
                                        readonly=True
                                       )
    price = fields.Float(required=True, default=0, readonly=True,
                         compute=compute_price,
                         store=True,
                         compute_sudo=True
                        )
    cost = fields.Float(required=True, default=0, readonly=True, compute=compute_cost, store=True, compute_sudo=True)
    effective_cost = fields.Float(required=True, default=0, readonly=True, store=True)
    ms_project_data = fields.Text()

    direct_sale_line_id = fields.Many2one('sale.order.line')
    product_id = fields.Many2one('product.product')
    sale_order_id = fields.Many2one('sale.order')
    sale_order_state = fields.Selection(related='sale_order_id.state')

    billing_plan = fields.Boolean(defaut=False)
    billing_project = fields.Many2one('project.project',
                                      compute=compute_billing_project,
                                      search=search_billing_project)
    can_be_invoiced = fields.Boolean(compute=compute_can_be_invoiced)
    invoiced = fields.Boolean(defaut=False)
    milestone = fields.Boolean(defaut=False)
    invoice_date = fields.Date()
    invoice_amount = fields.Float()

    invoice_id = fields.Many2one(
        'account.invoice'
    )

    @api.multi
    def markinvoiced(self):

        self.ensure_one()
        self.invoiced = True
        self.stage_id = self._stage_id_done

    @api.multi
    def marktoinvoice(self):

        self.ensure_one()
        self.invoiced = False
        self.stage_id = self._stage_id_fatturazione

    @api.multi
    def invoice(self):

        self.ensure_one()
        self.invoiced = True
        self.stage_id = self._stage_id_done

        view_id = self.env['ir.model.data'].get_object_reference('sale', 'view_sale_advance_payment_inv')[1]

        return {
            'type': 'ir.actions.act_window',
            'res_model': 'sale.advance.payment.inv',
            'view_mode': 'form',
            'view_type': 'form',
            'view_id': view_id,
            'target': 'new',
            'context': {'active_ids': [self.sale_order_id.id], 'active_model':'sale.order'}
        }

    # imposta prodotto ed offerta
    # se direct_sale_line_id popolato
    def _auto_set_product(self, values):

       ### auto set product if direct_sale_line_id
        if 'direct_sale_line_id' in values:

            product_id = False
            if values['direct_sale_line_id']:

                line = self.env['sale.order.line'].sudo().browse(
                            values['direct_sale_line_id']
                        )

                _logger.info("line IS %s" % line)
                _logger.info("product line IS %s" % line.product_id)

                product_id = line.product_id.id if line.product_id else False

                values['product_id'] = product_id
                values['sale_order_id'] = line.order_id.id

        # end auto set product

        return values

    @api.multi
    def write(self,values):
        """ override write """
        values = self.populate_billing_task(values, 'write')

        values = self._auto_set_product(values)

        """ DO NOT create procurment """
        res = super(Task, self).write(values)

        return res

    @api.model
    def create(self, values):
        """ override create """
        values = self.populate_billing_task(values, 'create')

        values = self._auto_set_product(values)

        return super(Task, self).create(values)

    def populate_billing_task(self, values, mode):
        """ gestione per righe di fatturazione
            nel caso di creazione task da ordine  """

        if('billing_plan' in values and values['billing_plan']):

            if(not 'date_deadline' in values or not values['date_deadline']):
                values['date_deadline'] = values['invoice_date']
            else:
                values['invoice_date'] = values['date_deadline']

            # nomi e date da ordine
            try:
                values['name'] = 'Fatturazione del %s' % parser.parse(values['invoice_date']).strftime("%d/%m/%Y")
            except:
                values['name'] = 'Fatturazione del %s' % values['invoice_date'].strftime("%d/%m/%Y")

            ## stato da fatturare (5002)
            ## se milestone altrimenti specifica (2)
            values['stage_id'] = self._stage_id_specifica if values['milestone'] else self._stage_id_fatturazione

            values['date_start'] = values['invoice_date']
            values['date_end'] = values['invoice_date']

            order = self.env['sale.order'].browse(values['sale_order_id'])
            config_analytic_account_administration = self.env['ir.config_parameter'].search([('key', '=', 'internal_analytic_account_administration_id')], limit=1)
            internal_analytic_account_administration = self.env['account.analytic.account'].browse([int(config_analytic_account_administration.value)])

            values['user_id'] = internal_analytic_account_administration.manager_id.id
            values['reviewer_id'] = internal_analytic_account_administration.manager_id.id

            ## se milestone tutti i rif sono al progetto
            if(values['milestone']):
                values['project_id'] = order.real_project_id.id
                values['project_ref_id'] = internal_analytic_account_administration.project_id.id
            else:
                values['project_id'] = internal_analytic_account_administration.project_id.id
                values['project_ref_id'] = order.real_project_id.id

        return values

    def onchange_date_deadline(
            self, cr, uid, ids, date_end, date_deadline, context=None):
        """ fix defect on date_deadline setup
            and date gant copy on date_ends """

        if not date_end or (date_end[:10] == self.browse(cr, uid, ids, context=context).date_deadline):

            # set hour at the end of the day but not too close to midnight
            date_end = date_deadline + ' 19:00:00' if date_deadline else False

        # retrieve possible date start from db
        date_start = self.browse(cr, uid, ids, context=context).date_start

        # if date_end is set
        # and date start is not set so that odoo fills the field with now
        # or the db value is > than date_end
        if(date_end and ((not date_start and parser.parse(date_end) < datetime.now())
                            or (date_start and parser.parse(date_start) > parser.parse(date_end)))):
            date_start = parser.parse(date_end) - timedelta(hours=1)

        # conditional output
        out = {'date_end': date_end}

        if(date_start):
            out['date_start'] = date_start

        return {'value': out}
<commit_msg>restore autoset<commit_after># -*- coding: utf-8 -*-
""" override task """

import pprint
import logging
from dateutil import parser
from datetime import datetime, date, timedelta

from openerp import models, fields, api, exceptions, tools
_logger = logging.getLogger(__name__)

class Task(models.Model):
    """ override task """

    _inherit = 'project.task'
    _stage_id_fatturazione = 5002
    _stage_id_specifica = 2
    _stage_id_annullo = 8
    _stage_id_done = 7

    @api.model
    def _needaction_domain_get(self):
        """ _needaction_domain_get remove unused counter on menu """
        return []

    @api.depends('sale_order_state', 'invoiced', 'stage_id')
    def compute_can_be_invoiced(self):
        """ compute_can_be_invoiced """
        for t in self:
            t.can_be_invoiced = (t.sale_order_state == 'manual' and
                                 (t.invoiced == True or not t.stage_id or t.stage_id.id not in [self._stage_id_fatturazione]))

    @api.depends('milestone', 'project_id', 'project_ref_id')
    def compute_billing_project(self):
        """ ricava il progetto di riferimento per la fatturazione """
        for task in self:
            if task.milestone:
                task.billing_project = task.project_id
            else:
                task.billing_project = task.project_ref_id

    def search_billing_project(self, operator, value):
        """ search_billing_project """
        if operator == 'like':
            operator = 'ilike'
        return [
            '|',
            ('project_ref_id.name', operator, value),
            ('project_id.name', operator, value)
            ]

    @api.depends('planned_hours', 'user_id')
    def compute_cost(self):
        """ ricava il costo orario dal contratto  """
        for task in self:
            resource = self.env['resource.resource'].sudo().search(
                [
                    ('user_id', '=', task.user_id.id)
                ]
            )
            employee = self.env['hr.employee'].sudo().search(
                [
                    ('resource_id', '=', resource.id)
                ]
            )
            cost = 0

            if resource and employee and employee.contract_id:
                hour_cost = employee.contract_id.wage
                cost = hour_cost * task.planned_hours

            task.cost = cost

    @api.depends('direct_sale_line_id', 'direct_sale_line_id.price_unit', 'cost', 'points')
    def compute_price(self):
        """ ricalcola il prezzo basato sui punti o sulla riga di offerta  """
        for task in self.sudo():
            if task.points:
                """
                ## based on points
                """
                task.price = task.an_acc_by_prj.point_unit_price * task.points
            else:
                """
                    ## base on sale offer
                """
                row_price = 0
                price = 0

                if(task.direct_sale_line_id and
                   isinstance(task.direct_sale_line_id.id, (int, long))):

                    line = task.direct_sale_line_id

                    row_price = line.price_unit * line.product_uom_qty
                    self._cr.execute("select sum(cost), case when count(id) > 0 then count(id) else 1 end  from project_task "
                                     " where direct_sale_line_id = %s and stage_id <> %s" %
                                     (line.id, self._stage_id_annullo))
                    record = self._cr.fetchone()

                    _logger.info(pprint.pformat(record))

                    """ costo calcolato in proporzione al prezzo dei task
                    o in proporzione al loro numero se il prezzo totale è 0 """
                    cost_factor = task.cost/float(record[0]) if record[0] else 1/float(record[1])

                    _logger.info("recalc cost/price %s" % cost_factor)

                    price = cost_factor * row_price

                task.price = price

    def _default_invoice_date_cache(self):
        if 'budget_line_cache_invoice_date' in self.cache:
            return self.cache['budget_line_cache_invoice_date']
        return False

    def _default_planned_amount_cache(self):
        if 'budget_line_cache_planned_amount' in self.cache:
            return self.cache['budget_line_cache_planned_amount']
        return False

    """
        ####
        ####  FIELDS
        ####
    """
    points = fields.Integer(string='Points')
    project_ref_id = fields.Many2one('project.project', 'Project reference')
    an_acc_by_prj = fields.Many2one('account.analytic.account',
                                    string="Contract/Analytic",
                                    related='project_id.analytic_account_id',
                                    readonly=True
                                   )
    an_acc_by_prj_ref = fields.Many2one('account.analytic.account',
                                        string="Contract/Analytic reference",
                                        related='project_ref_id.analytic_account_id',
                                        readonly=True
                                       )
    price = fields.Float(required=True, default=0, readonly=True,
                         compute=compute_price,
                         store=True,
                         compute_sudo=True
                        )
    cost = fields.Float(required=True, default=0, readonly=True, compute=compute_cost, store=True, compute_sudo=True)
    effective_cost = fields.Float(required=True, default=0, readonly=True, store=True)
    ms_project_data = fields.Text()

    direct_sale_line_id = fields.Many2one('sale.order.line')
    product_id = fields.Many2one('product.product')
    sale_order_id = fields.Many2one('sale.order')
    sale_order_state = fields.Selection(related='sale_order_id.state')

    billing_plan = fields.Boolean(defaut=False)
    billing_project = fields.Many2one('project.project',
                                      compute=compute_billing_project,
                                      search=search_billing_project)
    can_be_invoiced = fields.Boolean(compute=compute_can_be_invoiced)
    invoiced = fields.Boolean(defaut=False)
    milestone = fields.Boolean(defaut=False)
    invoice_date = fields.Date()
    invoice_amount = fields.Float()

    invoice_id = fields.Many2one(
        'account.invoice'
    )

    @api.multi
    def markinvoiced(self):

        self.ensure_one()
        self.invoiced = True
        self.stage_id = self._stage_id_done

    @api.multi
    def marktoinvoice(self):

        self.ensure_one()
        self.invoiced = False
        self.stage_id = self._stage_id_fatturazione

    @api.multi
    def invoice(self):

        self.ensure_one()
        self.invoiced = True
        self.stage_id = self._stage_id_done

        view_id = self.env['ir.model.data'].get_object_reference('sale', 'view_sale_advance_payment_inv')[1]

        return {
            'type': 'ir.actions.act_window',
            'res_model': 'sale.advance.payment.inv',
            'view_mode': 'form',
            'view_type': 'form',
            'view_id': view_id,
            'target': 'new',
            'context': {'active_ids': [self.sale_order_id.id], 'active_model':'sale.order'}
        }

    # imposta prodotto ed offerta
    # se direct_sale_line_id popolato
    def _auto_set_product(self, values):

        ### auto set product if direct_sale_line_id
        if 'direct_sale_line_id' in values:

            product_id = False
            if values['direct_sale_line_id']:

                line = self.env['sale.order.line'].sudo().browse(
                            values['direct_sale_line_id']
                        )

                _logger.info("line IS %s" % line)
                _logger.info("product line IS %s" % line.product_id)

                product_id = line.product_id.id if line.product_id else False

                values['product_id'] = product_id
                values['sale_order_id'] = line.order_id.id

        # end auto set product

        ### set auto invoiced
        if 'invoice_id' in values:

            values['invoiced'] = True if values['invoice_id'] else False
        # end auto invoiced

        return values

    @api.multi
    def write(self,values):
        """ override write """
        values = self.populate_billing_task(values, 'write')

        values = self._auto_set_product(values)

        """ DO NOT create procurment """
        res = super(Task, self).write(values)

        return res

    @api.model
    def create(self, values):
        """ override create """
        values = self.populate_billing_task(values, 'create')

        values = self._auto_set_product(values)

        return super(Task, self).create(values)

    def populate_billing_task(self, values, mode):
        """ gestione per righe di fatturazione
            nel caso di creazione task da ordine  """

        if('billing_plan' in values and values['billing_plan']):

            if(not 'date_deadline' in values or not values['date_deadline']):
                values['date_deadline'] = values['invoice_date']
            else:
                values['invoice_date'] = values['date_deadline']

            # nomi e date da ordine
            try:
                values['name'] = 'Fatturazione del %s' % parser.parse(values['invoice_date']).strftime("%d/%m/%Y")
            except:
                values['name'] = 'Fatturazione del %s' % values['invoice_date'].strftime("%d/%m/%Y")

            ## stato da fatturare (5002)
            ## se milestone altrimenti specifica (2)
            values['stage_id'] = self._stage_id_specifica if values['milestone'] else self._stage_id_fatturazione

            values['date_start'] = values['invoice_date']
            values['date_end'] = values['invoice_date']

            order = self.env['sale.order'].browse(values['sale_order_id'])
            config_analytic_account_administration = self.env['ir.config_parameter'].search([('key', '=', 'internal_analytic_account_administration_id')], limit=1)
            internal_analytic_account_administration = self.env['account.analytic.account'].browse([int(config_analytic_account_administration.value)])

            values['user_id'] = internal_analytic_account_administration.manager_id.id
            values['reviewer_id'] = internal_analytic_account_administration.manager_id.id

            ## se milestone tutti i rif sono al progetto
            if(values['milestone']):
                values['project_id'] = order.real_project_id.id
                values['project_ref_id'] = internal_analytic_account_administration.project_id.id
            else:
                values['project_id'] = internal_analytic_account_administration.project_id.id
                values['project_ref_id'] = order.real_project_id.id

        return values

    def onchange_date_deadline(
            self, cr, uid, ids, date_end, date_deadline, context=None):
        """ fix defect on date_deadline setup
            and date gant copy on date_ends """

        if not date_end or (date_end[:10] == self.browse(cr, uid, ids, context=context).date_deadline):

            # set hour at the end of the day but not too close to midnight
            date_end = date_deadline + ' 19:00:00' if date_deadline else False

        # retrieve possible date start from db
        date_start = self.browse(cr, uid, ids, context=context).date_start

        # if date_end is set
        # and date start is not set so that odoo fills the field with now
        # or the db value is > than date_end
        if(date_end and ((not date_start and parser.parse(date_end) < datetime.now())
                            or (date_start and parser.parse(date_start) > parser.parse(date_end)))):
            date_start = parser.parse(date_end) - timedelta(hours=1)

        # conditional output
        out = {'date_end': date_end}

        if(date_start):
            out['date_start'] = date_start

        return {'value': out}
<|endoftext|>


<commit_before>__author__ = 'michaelcaraccio'

import pygame
from models.city import City
from models.point import Point
from models.problem import Problem
from pygame.locals import KEYDOWN, QUIT, MOUSEBUTTONDOWN, K_RETURN, K_ESCAPE

def initScreen():

    pygame.init()
    window = pygame.display.set_mode((screen_x, screen_y))
    pygame.display.set_caption('Exemple')
    screen = pygame.display.get_surface()
    font = pygame.font.Font(None, 30)

    return screen, window, font

def draw(problem):
    screen.fill(0)
    citylist = problem.getCities()
    pointlist = []

    # On affiche chaque ville sur l'écran
    for city in citylist:
        pos = city.coords()

        # Tableau de points - Pour la création de ligne
        pointlist.append((pos.x(), pos.y()))

        pygame.draw.circle(screen, city_color, (pos.x(), pos.y()), city_radius)
        myfont = pygame.font.Font(None, 20)

        # Création du label contenant le nom de la ville
        label = myfont.render(city.name(), 1, text_color, None)

        # Affichage du label
        screen.blit(label, (pos.x() + 10, pos.y() - 6))

    # Sans oublier de mettre le premier élément à la dernière position
    pointlist.append((pointlist[0]))

     # On créer les lignes entre les villes
    pygame.draw.lines(screen, 0xffffffff, False, [p for p in pointlist], 2)

    text = font.render("Nombre: %i" % problem.getSize(), True, font_color)
    textRect = text.get_rect()
    screen.blit(text, textRect)
    pygame.display.flip()

if __name__ == "__main__":

    # Quelques paramètres
    screen_x = 500
    screen_y = 500
    city_color = [0, 255, 0]  # Green
    text_color = [0, 255, 0]  # Green
    city_radius = 5
    font_color = [255, 255, 255]  # white

    # Initialisation de la fenêtre, window et font
    screen, window, font = initScreen()

    city1 = City(Point(200, 300), "1")
    city2 = City(Point(300, 400), "2")
    city3 = City(Point(333, 50), "3")
    problem = Problem([city1, city2, city3])

    # Demande d'affichage de toutes les villes sur l'écran
    draw(problem)

    # Refresh de l'écran
    pygame.display.flip()

    while True:
        event = pygame.event.wait()
        if event.type == pygame.QUIT:
            pygame.quit()
            break
        if event.type == KEYDOWN:
            pygame.quit()
            break<commit_msg>Affiche des solutions au lieu de probleme<commit_after>__author__ = 'michaelcaraccio'

import pygame
from models.city import City
from models.point import Point
from models.problem import Problem
from models.solution import Solution
from pygame.locals import KEYDOWN, QUIT, MOUSEBUTTONDOWN, K_RETURN, K_ESCAPE

def initScreen():

    pygame.init()
    window = pygame.display.set_mode((screen_x, screen_y))
    pygame.display.set_caption('Exemple')
    screen = pygame.display.get_surface()
    font = pygame.font.Font(None, 30)

    return screen, window, font

def draw(solution):
    screen.fill(0)
    citylist = solution
    pointlist = []

    # On affiche chaque ville sur l'écran
    for city in citylist:
        pos = city.coords()

        # Tableau de points - Pour la création de ligne
        pointlist.append((pos.x(), pos.y()))

        pygame.draw.circle(screen, city_color, (pos.x(), pos.y()), city_radius)
        myfont = pygame.font.Font(None, 20)

        # Création du label contenant le nom de la ville
        label = myfont.render(city.name(), 1, text_color, None)

        # Affichage du label
        screen.blit(label, (pos.x() + 10, pos.y() - 6))

    # Sans oublier de mettre le premier élément à la dernière position
    pointlist.append((pointlist[0]))

     # On créer les lignes entre les villes
    pygame.draw.lines(screen, 0xffffffff, False, [p for p in pointlist], 2)

    text = font.render("Nombre: %i" % problem.getSize(), True, font_color)
    textRect = text.get_rect()
    screen.blit(text, textRect)
    pygame.display.flip()

if __name__ == "__main__":

    # Quelques paramètres
    screen_x = 500
    screen_y = 500
    city_color = [0, 255, 0]  # Green
    text_color = [0, 255, 0]  # Green
    city_radius = 5
    font_color = [255, 255, 255]  # white

    # Initialisation de la fenêtre, window et font
    screen, window, font = initScreen()

    city1 = City(Point(200, 300), "1")
    city2 = City(Point(300, 400), "2")
    city3 = City(Point(333, 50), "3")
    problem = Problem([city1, city2, city3])
    soluce = Solution(problem)

    # Demande d'affichage de toutes les villes sur l'écran
    draw(soluce._path)

    # Refresh de l'écran
    pygame.display.flip()

    while True:
        event = pygame.event.wait()
        if event.type == pygame.QUIT:
            pygame.quit()
            break
        if event.type == KEYDOWN:
            pygame.quit()
            break<|endoftext|>


<commit_before>import string
import regex
from collections import deque
from flanker.mime.message.headers import encodedword, parametrized
from flanker.mime.message.headers.wrappers import ContentType, WithParams
from flanker.mime.message.errors import DecodingError
from flanker.mime.message.utils import to_unicode
from flanker.utils import is_pure_ascii

_RE_HEADER = regex.compile(r'^(From |[\041-\071\073-\176]+:|[\t ])')

def normalize(header_name):
    return string.capwords(header_name.lower(), '-')

def parse_stream(stream):
    """Reads the incoming stream and returns list of tuples"""
    out = deque()
    for header in _unfold_header_lines(_read_header_lines(stream)):
        out.append(parse_header(header))

    return out

def parse_header(header):
    """ Accepts a raw header with name, colons and newlines
    and returns it's parsed value
    """
    name, val = _split_header(header)
    if not is_pure_ascii(name):
        raise DecodingError('Non-ascii header name')

    return name, parse_header_value(name, encodedword.unfold(val))

def parse_header_value(name, val):
    if not is_pure_ascii(val):
        val = to_unicode(val)
    if parametrized.is_parametrized(name, val):
        val, params = parametrized.decode(val)
        if val is not None and not is_pure_ascii(val):
            raise DecodingError('Non-ascii content header value')
        if name == 'Content-Type':
            main, sub = parametrized.fix_content_type(val)
            return ContentType(main, sub, params)

        return WithParams(val, params)

    return val

def is_empty(line):
    return line in ('\r\n', '\r', '\n')

def _read_header_lines(fp):
    """Read lines with headers until the start of body"""
    lines = deque()
    for line in fp:
        if is_empty(line):
            break

        # tricky case if it's not a header and not an empty line
        # ususally means that user forgot to separate the body and newlines
        # so "unread" this line here, what means to treat it like a body
        if not _RE_HEADER.match(line):
            fp.seek(fp.tell() - len(line))
            break

        lines.append(line)

    return lines

def _unfold_header_lines(lines):
    headers = deque()
    for line in lines:
        # ignore unix from
        if line.startswith('From '):
            continue

        # this is continuation
        if line[0] in ' \t':
            _extend_last_header(headers, line)
            continue

        headers.append(line)

    new_headers = deque()
    for h in headers:
        if isinstance(h, deque):
            new_headers.append(''.join(h).rstrip('\r\n'))
            continue

        new_headers.append(h.rstrip('\r\n'))

    return new_headers

def _extend_last_header(headers, line):
    # Ignore continuation header lines at the top of a part.
    if len(headers) == 0:
        return

    header = headers.pop()

    # If the latest header is already multiline then extend it.
    if isinstance(header, deque):
        header.append(line)
        headers.append(header)
        return

    # Convert the latest header to a multiline one.
    headers.append(deque((header, line)))

def _split_header(header):
    pair = header.split(':', 1)
    if len(pair) == 2:
        return normalize(pair[0].rstrip()), pair[1].lstrip()

    return None, None
<commit_msg>Fix simple typo: ususally -> usually<commit_after>import string
import regex
from collections import deque
from flanker.mime.message.headers import encodedword, parametrized
from flanker.mime.message.headers.wrappers import ContentType, WithParams
from flanker.mime.message.errors import DecodingError
from flanker.mime.message.utils import to_unicode
from flanker.utils import is_pure_ascii

_RE_HEADER = regex.compile(r'^(From |[\041-\071\073-\176]+:|[\t ])')

def normalize(header_name):
    return string.capwords(header_name.lower(), '-')

def parse_stream(stream):
    """Reads the incoming stream and returns list of tuples"""
    out = deque()
    for header in _unfold_header_lines(_read_header_lines(stream)):
        out.append(parse_header(header))

    return out

def parse_header(header):
    """ Accepts a raw header with name, colons and newlines
    and returns it's parsed value
    """
    name, val = _split_header(header)
    if not is_pure_ascii(name):
        raise DecodingError('Non-ascii header name')

    return name, parse_header_value(name, encodedword.unfold(val))

def parse_header_value(name, val):
    if not is_pure_ascii(val):
        val = to_unicode(val)
    if parametrized.is_parametrized(name, val):
        val, params = parametrized.decode(val)
        if val is not None and not is_pure_ascii(val):
            raise DecodingError('Non-ascii content header value')
        if name == 'Content-Type':
            main, sub = parametrized.fix_content_type(val)
            return ContentType(main, sub, params)

        return WithParams(val, params)

    return val

def is_empty(line):
    return line in ('\r\n', '\r', '\n')

def _read_header_lines(fp):
    """Read lines with headers until the start of body"""
    lines = deque()
    for line in fp:
        if is_empty(line):
            break

        # tricky case if it's not a header and not an empty line
        # usually means that user forgot to separate the body and newlines
        # so "unread" this line here, what means to treat it like a body
        if not _RE_HEADER.match(line):
            fp.seek(fp.tell() - len(line))
            break

        lines.append(line)

    return lines

def _unfold_header_lines(lines):
    headers = deque()
    for line in lines:
        # ignore unix from
        if line.startswith('From '):
            continue

        # this is continuation
        if line[0] in ' \t':
            _extend_last_header(headers, line)
            continue

        headers.append(line)

    new_headers = deque()
    for h in headers:
        if isinstance(h, deque):
            new_headers.append(''.join(h).rstrip('\r\n'))
            continue

        new_headers.append(h.rstrip('\r\n'))

    return new_headers

def _extend_last_header(headers, line):
    # Ignore continuation header lines at the top of a part.
    if len(headers) == 0:
        return

    header = headers.pop()

    # If the latest header is already multiline then extend it.
    if isinstance(header, deque):
        header.append(line)
        headers.append(header)
        return

    # Convert the latest header to a multiline one.
    headers.append(deque((header, line)))

def _split_header(header):
    pair = header.split(':', 1)
    if len(pair) == 2:
        return normalize(pair[0].rstrip()), pair[1].lstrip()

    return None, None
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import threading

import SimpleXMLRPCServer, signal, sys, xmlrpclib
import SocketServer
import socket
import logging
import logging.handlers
import os

_service = {}
_group = {}
_res_id = 1
_res = {}

class ServiceEndPointCall(object):
    def __init__(self, id, method):
        self._id = id
        self._meth = method

    def __call__(self, *args):
        _res[self._id] = self._meth(*args)
        return self._id

class ServiceEndPoint(object):
    def __init__(self, name, id):
        self._id = id
        self._meth = {}
        s = _service[name]
        for m in s._method:
            self._meth[m] = s._method[m]

    def __getattr__(self, name):
        return ServiceEndPointCall(self._id, self._meth[name])

class Service(object):
    _serviceEndPointID = 0

    def __init__(self, name, audience=''):
        _service[name] = self
        self.__name = name
        self._method = {}
        self.exportedMethods = None
        self._response_process = None
        self._response_process_id = None
        self._response = None

    def joinGroup(self, name):
        if not name in _group:
            _group[name] = {}
        _group[name][self.__name] = self

    def exportMethod(self, m):
        if callable(m):
            self._method[m.__name__] = m

    def serviceEndPoint(self, s):
        if Service._serviceEndPointID >= 2**16:
            Service._serviceEndPointID = 0
        Service._serviceEndPointID += 1
        return ServiceEndPoint(s, self._serviceEndPointID)

    def conversationId(self):
        return 1

    def processResponse(self, s, id):
        self._response_process, self._response_process_id = s, id

    def processFailure(self, s, id):
        pass

    def resumeResponse(self, s):
        pass

    def cancelResponse(self, s):
        pass

    def suspendResponse(self, s):
        if self._response_process:
            self._response_process(self._response_process_id,
                                   _res[self._response_process_id])
        self._response_process = None
        self._response = s(self._response_process_id)

    def abortResponse(self, error, description, origin, details):
        import tools
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

    def currentFailure(self, s):
        pass

class LocalService(Service):
    def __init__(self, name):
        self.__name = name
        try:
            s = _service[name]
            self._service = s
            for m in s._method:
                setattr(self, m, s._method[m])
        except KeyError, keyError:
            Logger().notifyChannel('module', LOG_ERROR, 'This service does not exists: %s' % (str(keyError),) )
            raise

class ServiceUnavailable(Exception):
    pass

def service_exist(name):
    return (name in _service) and bool(_service[name])

def get_rpc_paths():
    return map(lambda s: '/xmlrpc/%s' % s, _service)

LOG_DEBUG = 'debug'
LOG_INFO = 'info'
LOG_WARNING = 'warn'
LOG_ERROR = 'error'
LOG_CRITICAL = 'critical'

def init_logger():
    from tools import config
    import os

    if config['logfile']:
        logf = config['logfile']
        # test if the directories exist, else create them
        try:
            dirname = os.path.dirname(logf)
            if not os.path.isdir(dirname):
                res = os.makedirs(dirname)
            handler = logging.handlers.TimedRotatingFileHandler(logf,'D',1,30)
        except:
            sys.stderr.write("ERROR: couldn't create the logfile directory\n")
            handler = logging.StreamHandler(sys.stdout)
    else:
        handler = logging.StreamHandler(sys.stdout)

    # create a format for log messages and dates
    formatter = logging.Formatter('[%(asctime)s] %(levelname)s:%(name)s:%(message)s', '%a %b %d %H:%M:%S %Y')

    # tell the handler to use this format
    handler.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger().addHandler(handler)
    logging.getLogger().setLevel(logging.INFO)

    if isinstance(handler, logging.StreamHandler) and os.name != 'nt':
        # change color of level names
        # uses of ANSI color codes
        # see http://pueblo.sourceforge.net/doc/manual/ansi_color_codes.html
        # maybe use http://code.activestate.com/recipes/574451/
        colors = ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', None, 'default']
        foreground = lambda f: 30 + colors.index(f)
        background = lambda f: 40 + colors.index(f)

        mapping = {
            'DEBUG': ('blue', 'default'),
            'INFO': ('green', 'default'),
            'WARNING': ('yellow', 'default'),
            'ERROR': ('red', 'default'),
            'CRITICAL': ('white', 'red'),
        }

        for level, (fg, bg) in mapping.items():
            msg = "\x1b[%dm\x1b[%dm%s\x1b[0m" % (foreground(fg), background(bg), level)
            logging.addLevelName(getattr(logging, level), msg)

class Logger(object):
    def notifyChannel(self, name, level, msg):
        log = logging.getLogger(name)
        getattr(log, level)(msg)

init_logger()

class Agent(object):
    _timers = []
    _logger = Logger()

    def setAlarm(self, fn, dt, args=None, kwargs=None):
        if not args:
            args = []
        if not kwargs:
            kwargs = {}
        wait = dt - time.time()
        if wait > 0:
            self._logger.notifyChannel('timers', LOG_DEBUG, "Job scheduled in %s seconds for %s.%s" % (wait, fn.im_class.__name__, fn.func_name))
            timer = threading.Timer(wait, fn, args, kwargs)
            timer.start()
            self._timers.append(timer)
        for timer in self._timers[:]:
            if not timer.isAlive():
                self._timers.remove(timer)

    def quit(cls):
        for timer in cls._timers:
            timer.cancel()
    quit = classmethod(quit)

class RpcGateway(object):
    def __init__(self, name):
        self.name = name

class Dispatcher(object):
    def __init__(self):
        pass

    def monitor(self, signal):
        pass

    def run(self):
        pass

class xmlrpc(object):
    class RpcGateway(object):
        def __init__(self, name):
            self.name = name

class GenericXMLRPCRequestHandler:
    def _dispatch(self, method, params):
        #print 'TERP-CALL : ',method, params
        import traceback
        try:
            n = self.path.split("/")[-1]
            s = LocalService(n)
            m = getattr(s, method)
            s._service._response = None
            r = m(*params)
            res = s._service._response
            if res != None:
                r = res
            return r
        except Exception, e:
            tb_s = reduce(lambda x, y: x+y, traceback.format_exception(
                sys.exc_type, sys.exc_value, sys.exc_traceback))
            s = str(e)
            import tools
            if tools.config['debug_mode']:
                import pdb
                tb = sys.exc_info()[2]
                pdb.post_mortem(tb)
            raise xmlrpclib.Fault(s, tb_s)

class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
    SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.rpc_paths = get_rpc_paths()

class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer.SimpleXMLRPCServer):

    def server_bind(self):
        try:
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)
        except:
            Logger().notifyChannel('init', LOG_ERROR, 'Address already in use')
            sys.exit(1)

class HttpDaemon(threading.Thread):

    def __init__(self, interface, port, secure=False):
        threading.Thread.__init__(self)
        self.__port = port
        self.__interface = interface
        self.secure = secure
        if secure:
            from ssl import SecureXMLRPCServer

            class SecureXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SecureXMLRPCServer.SecureXMLRPCRequestHandler):
                SecureXMLRPCServer.SecureXMLRPCRequestHandler.rpc_paths = get_rpc_paths()

            class SecureThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SecureXMLRPCServer.SecureXMLRPCServer):
                def server_bind(self):
                    try:
                        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                        SecureXMLRPCServer.SecureXMLRPCServer.server_bind(self)
                    except:
                        sys.stderr.write("ERROR: address already in use\n")
                        sys.exit(1)

            self.server = SecureThreadedXMLRPCServer((interface, port),
                    SecureXMLRPCRequestHandler, 0)
        else:
            self.server = SimpleThreadedXMLRPCServer((interface, port),
                    SimpleXMLRPCRequestHandler, 0)

    def attach(self, path, gw):
        pass

    def stop(self):
        self.running = False
        if os.name != 'nt':
            if hasattr(socket, 'SHUT_RDWR'):
                if self.secure:
                    self.server.socket.sock_shutdown(socket.SHUT_RDWR)
                else:
                    self.server.socket.shutdown(socket.SHUT_RDWR)
            else:
                if self.secure:
                    self.server.socket.sock_shutdown(2)
                else:
                    self.server.socket.shutdown(2)
        self.server.socket.close()

    def run(self):
        self.server.register_introspection_functions()

        self.running = True
        while self.running:
            self.server.handle_request()
        return True

        # If the server need to be run recursively
        #
        #signal.signal(signal.SIGALRM, self.my_handler)
        #signal.alarm(6)
        #while True:
        #   self.server.handle_request()
        #signal.alarm(0)          # Disable the alarm

import tiny_socket
class TinySocketClientThread(threading.Thread):
    def __init__(self, sock, threads):
        threading.Thread.__init__(self)
        self.sock = sock
        self.threads = threads

    def run(self):
        import traceback
        import time
        import select
        self.running = True
        try:
            ts = tiny_socket.mysocket(self.sock)
        except:
            self.sock.close()
            self.threads.remove(self)
            return False
        while self.running:
            try:
                msg = ts.myreceive()
            except:
                self.sock.close()
                self.threads.remove(self)
                return False
            try:
                s = LocalService(msg[0])
                m = getattr(s, msg[1])
                s._service._response = None
                r = m(*msg[2:])
                res = s._service._response
                if res != None:
                    r = res
                ts.mysend(r)
            except Exception, e:
                tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
                s = str(e)
                import tools
                if tools.config['debug_mode']:
                    import pdb
                    tb = sys.exc_info()[2]
                    pdb.post_mortem(tb)
                ts.mysend(e, exception=True, traceback=tb_s)
            except:
                pass
            self.sock.close()
            self.threads.remove(self)
            return True

    def stop(self):
        self.running = False

class TinySocketServerThread(threading.Thread):
    def __init__(self, interface, port, secure=False):
        threading.Thread.__init__(self)
        self.__port = port
        self.__interface = interface
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind((self.__interface, self.__port))
        self.socket.listen(5)
        self.threads = []

    def run(self):
        import select
        try:
            self.running = True
            while self.running:
                (clientsocket, address) = self.socket.accept()
                ct = TinySocketClientThread(clientsocket, self.threads)
                self.threads.append(ct)
                ct.start()
            self.socket.close()
        except Exception, e:
            self.socket.close()
            return False

    def stop(self):
        self.running = False
        for t in self.threads:
            t.stop()
        try:
            if hasattr(socket, 'SHUT_RDWR'):
                self.socket.shutdown(socket.SHUT_RDWR)
            else:
                self.socket.shutdown(2)
            self.socket.close()
        except:
            return False

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[IMP] Change the presentation of the logger<commit_after>#!/usr/bin/python
# -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import threading

import SimpleXMLRPCServer, signal, sys, xmlrpclib
import SocketServer
import socket
import logging
import logging.handlers
import os

_service = {}
_group = {}
_res_id = 1
_res = {}

class ServiceEndPointCall(object):
    def __init__(self, id, method):
        self._id = id
        self._meth = method

    def __call__(self, *args):
        _res[self._id] = self._meth(*args)
        return self._id

class ServiceEndPoint(object):
    def __init__(self, name, id):
        self._id = id
        self._meth = {}
        s = _service[name]
        for m in s._method:
            self._meth[m] = s._method[m]

    def __getattr__(self, name):
        return ServiceEndPointCall(self._id, self._meth[name])

class Service(object):
    _serviceEndPointID = 0

    def __init__(self, name, audience=''):
        _service[name] = self
        self.__name = name
        self._method = {}
        self.exportedMethods = None
        self._response_process = None
        self._response_process_id = None
        self._response = None

    def joinGroup(self, name):
        if not name in _group:
            _group[name] = {}
        _group[name][self.__name] = self

    def exportMethod(self, m):
        if callable(m):
            self._method[m.__name__] = m

    def serviceEndPoint(self, s):
        if Service._serviceEndPointID >= 2**16:
            Service._serviceEndPointID = 0
        Service._serviceEndPointID += 1
        return ServiceEndPoint(s, self._serviceEndPointID)

    def conversationId(self):
        return 1

    def processResponse(self, s, id):
        self._response_process, self._response_process_id = s, id

    def processFailure(self, s, id):
        pass

    def resumeResponse(self, s):
        pass

    def cancelResponse(self, s):
        pass

    def suspendResponse(self, s):
        if self._response_process:
            self._response_process(self._response_process_id,
                                   _res[self._response_process_id])
        self._response_process = None
        self._response = s(self._response_process_id)

    def abortResponse(self, error, description, origin, details):
        import tools
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

    def currentFailure(self, s):
        pass

class LocalService(Service):
    def __init__(self, name):
        self.__name = name
        try:
            s = _service[name]
            self._service = s
            for m in s._method:
                setattr(self, m, s._method[m])
        except KeyError, keyError:
            Logger().notifyChannel('module', LOG_ERROR, 'This service does not exists: %s' % (str(keyError),) )
            raise

class ServiceUnavailable(Exception):
    pass

def service_exist(name):
    return (name in _service) and bool(_service[name])

def get_rpc_paths():
    return map(lambda s: '/xmlrpc/%s' % s, _service)

LOG_DEBUG = 'debug'
LOG_INFO = 'info'
LOG_WARNING = 'warn'
LOG_ERROR = 'error'
LOG_CRITICAL = 'critical'

def init_logger():
    from tools import config
    import os

    if config['logfile']:
        logf = config['logfile']
        # test if the directories exist, else create them
        try:
            dirname = os.path.dirname(logf)
            if not os.path.isdir(dirname):
                res = os.makedirs(dirname)
            handler = logging.handlers.TimedRotatingFileHandler(logf,'D',1,30)
        except:
            sys.stderr.write("ERROR: couldn't create the logfile directory\n")
            handler = logging.StreamHandler(sys.stdout)
    else:
        handler = logging.StreamHandler(sys.stdout)

    # create a format for log messages and dates
    formatter = logging.Formatter('[%(asctime)s] %(levelname)s:%(name)s:%(message)s', '%a %b %d %H:%M:%S %Y')

    # tell the handler to use this format
    handler.setFormatter(formatter)

    # add the handler to the root logger
    logging.getLogger().addHandler(handler)
    logging.getLogger().setLevel(logging.INFO)

    if isinstance(handler, logging.StreamHandler) and os.name != 'nt':
        # change color of level names
        # uses of ANSI color codes
        # see http://pueblo.sourceforge.net/doc/manual/ansi_color_codes.html
        # maybe use http://code.activestate.com/recipes/574451/
        colors = ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', None, 'default']
        foreground = lambda f: 30 + colors.index(f)
        background = lambda f: 40 + colors.index(f)

        mapping = {
            'DEBUG': ('blue', 'default'),
            'INFO': ('green', 'default'),
            'WARNING': ('yellow', 'default'),
            'ERROR': ('red', 'default'),
            'CRITICAL': ('white', 'red'),
        }

        for level, (fg, bg) in mapping.items():
            msg = "\x1b[%dm\x1b[%dm%s\x1b[0m" % (foreground(fg), background(bg), level)
            logging.addLevelName(getattr(logging, level), msg)

class Logger(object):
    def notifyChannel(self, name, level, msg):
        log = logging.getLogger(name)
        level_method = getattr(log, level)

        result = msg.strip().split('\n')
        if len(result)>1:
            for idx, s in enumerate(result):
                level_method('[%02d]: %s' % (idx+1, s,))
        elif result:
            level_method(result[0])

init_logger()

class Agent(object):
    _timers = []
    _logger = Logger()

    def setAlarm(self, fn, dt, args=None, kwargs=None):
        if not args:
            args = []
        if not kwargs:
            kwargs = {}
        wait = dt - time.time()
        if wait > 0:
            self._logger.notifyChannel('timers', LOG_DEBUG, "Job scheduled in %s seconds for %s.%s" % (wait, fn.im_class.__name__, fn.func_name))
            timer = threading.Timer(wait, fn, args, kwargs)
            timer.start()
            self._timers.append(timer)
        for timer in self._timers[:]:
            if not timer.isAlive():
                self._timers.remove(timer)

    def quit(cls):
        for timer in cls._timers:
            timer.cancel()
    quit = classmethod(quit)

class RpcGateway(object):
    def __init__(self, name):
        self.name = name

class Dispatcher(object):
    def __init__(self):
        pass

    def monitor(self, signal):
        pass

    def run(self):
        pass

class xmlrpc(object):
    class RpcGateway(object):
        def __init__(self, name):
            self.name = name

class GenericXMLRPCRequestHandler:
    def _dispatch(self, method, params):
        #print 'TERP-CALL : ',method, params
        import traceback
        try:
            n = self.path.split("/")[-1]
            s = LocalService(n)
            m = getattr(s, method)
            s._service._response = None
            r = m(*params)
            res = s._service._response
            if res != None:
                r = res
            return r
        except Exception, e:
            tb_s = reduce(lambda x, y: x+y, traceback.format_exception(
                sys.exc_type, sys.exc_value, sys.exc_traceback))
            s = str(e)
            import tools
            if tools.config['debug_mode']:
                import pdb
                tb = sys.exc_info()[2]
                pdb.post_mortem(tb)
            raise xmlrpclib.Fault(s, tb_s)

class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
    SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.rpc_paths = get_rpc_paths()

class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer.SimpleXMLRPCServer):

    def server_bind(self):
        try:
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self)
        except:
            Logger().notifyChannel('init', LOG_ERROR, 'Address already in use')
            sys.exit(1)

class HttpDaemon(threading.Thread):

    def __init__(self, interface, port, secure=False):
        threading.Thread.__init__(self)
        self.__port = port
        self.__interface = interface
        self.secure = secure
        if secure:
            from ssl import SecureXMLRPCServer

            class SecureXMLRPCRequestHandler(GenericXMLRPCRequestHandler, SecureXMLRPCServer.SecureXMLRPCRequestHandler):
                SecureXMLRPCServer.SecureXMLRPCRequestHandler.rpc_paths = get_rpc_paths()

            class SecureThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SecureXMLRPCServer.SecureXMLRPCServer):
                def server_bind(self):
                    try:
                        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                        SecureXMLRPCServer.SecureXMLRPCServer.server_bind(self)
                    except:
                        sys.stderr.write("ERROR: address already in use\n")
                        sys.exit(1)

            self.server = SecureThreadedXMLRPCServer((interface, port),
                    SecureXMLRPCRequestHandler, 0)
        else:
            self.server = SimpleThreadedXMLRPCServer((interface, port),
                    SimpleXMLRPCRequestHandler, 0)

    def attach(self, path, gw):
        pass

    def stop(self):
        self.running = False
        if os.name != 'nt':
            if hasattr(socket, 'SHUT_RDWR'):
                if self.secure:
                    self.server.socket.sock_shutdown(socket.SHUT_RDWR)
                else:
                    self.server.socket.shutdown(socket.SHUT_RDWR)
            else:
                if self.secure:
                    self.server.socket.sock_shutdown(2)
                else:
                    self.server.socket.shutdown(2)
        self.server.socket.close()

    def run(self):
        self.server.register_introspection_functions()

        self.running = True
        while self.running:
            self.server.handle_request()
        return True

        # If the server need to be run recursively
        #
        #signal.signal(signal.SIGALRM, self.my_handler)
        #signal.alarm(6)
        #while True:
        #   self.server.handle_request()
        #signal.alarm(0)          # Disable the alarm

import tiny_socket
class TinySocketClientThread(threading.Thread):
    def __init__(self, sock, threads):
        threading.Thread.__init__(self)
        self.sock = sock
        self.threads = threads

    def run(self):
        import traceback
        import time
        import select
        self.running = True
        try:
            ts = tiny_socket.mysocket(self.sock)
        except:
            self.sock.close()
            self.threads.remove(self)
            return False
        while self.running:
            try:
                msg = ts.myreceive()
            except:
                self.sock.close()
                self.threads.remove(self)
                return False
            try:
                s = LocalService(msg[0])
                m = getattr(s, msg[1])
                s._service._response = None
                r = m(*msg[2:])
                res = s._service._response
                if res != None:
                    r = res
                ts.mysend(r)
            except Exception, e:
                tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
                s = str(e)
                import tools
                if tools.config['debug_mode']:
                    import pdb
                    tb = sys.exc_info()[2]
                    pdb.post_mortem(tb)
                ts.mysend(e, exception=True, traceback=tb_s)
            except:
                pass
            self.sock.close()
            self.threads.remove(self)
            return True

    def stop(self):
        self.running = False

class TinySocketServerThread(threading.Thread):
    def __init__(self, interface, port, secure=False):
        threading.Thread.__init__(self)
        self.__port = port
        self.__interface = interface
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind((self.__interface, self.__port))
        self.socket.listen(5)
        self.threads = []

    def run(self):
        import select
        try:
            self.running = True
            while self.running:
                (clientsocket, address) = self.socket.accept()
                ct = TinySocketClientThread(clientsocket, self.threads)
                self.threads.append(ct)
                ct.start()
            self.socket.close()
        except Exception, e:
            self.socket.close()
            return False

    def stop(self):
        self.running = False
        for t in self.threads:
            t.stop()
        try:
            if hasattr(socket, 'SHUT_RDWR'):
                self.socket.shutdown(socket.SHUT_RDWR)
            else:
                self.socket.shutdown(2)
            self.socket.close()
        except:
            return False

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>from __future__ import absolute_import

from collections import OrderedDict, Sequence, namedtuple
from itertools import combinations
from time import time

import numpy as np

import cgen as c

from devito.dimension import Dimension
from devito.dle.inspection import retrieve_iteration_tree
from devito.dle.manipulation import compose_nodes
from devito.dse import terminals
from devito.interfaces import ScalarData, SymbolicData
from devito.logger import dle, dle_warning
from devito.nodes import Element, Function, Iteration, List
from devito.tools import as_tuple, flatten
from devito.visitors import FindSections, FindSymbols, IsPerfectIteration, Transformer

def transform(node, mode='basic'):
    """
    Transform Iteration/Expression trees to generate highly optimized C code.

    :param node: The Iteration/Expression tree to be transformed, or an iterable
                 of Iteration/Expression trees.
    :param mode: Drive the tree transformation. Available modes are: ::

                    * 'noop': Do nothing.
                    * 'blocking': Apply loop blocking
                    * 'split': Identify and split elemental functions
                    * 'advanced': Split elemental functions and apply loop blocking.
    """

    if isinstance(node, Sequence):
        assert all(n.is_Node for n in node)
        node = list(node)
    elif node.is_Node:
        node = [node]
    else:
        raise ValueError("Got illegal node of type %s." % type(node))

    if not mode:
        return State(node)
    elif isinstance(mode, str):
        mode = set([mode])
    else:
        try:
            mode = set(mode)
        except TypeError:
            dle_warning("Arg mode must be str or tuple (got %s)" % type(mode))
            return State(node)
    if mode.isdisjoint({'noop', 'blocking', 'split', 'advanced'}):
        dle_warning("Unknown transformer mode(s) %s" % str(mode))
        return State(node)
    else:
        return Rewriter(node).run(mode)

    return Rewriter(node).run(mode)

def dle_transformation(func):

    def wrapper(self, state, **kwargs):
        if kwargs['mode'].intersection(set(self.triggers[func.__name__])):
            tic = time()
            state.update(**func(self, state))
            toc = time()

            key = '%s%d' % (func.__name__, len(self.timings))
            self.timings[key] = toc - tic

    return wrapper

class State(object):

    """Represent the output of the DLE."""

    def __init__(self, nodes):
        self.nodes = as_tuple(nodes)

        self.elemental_functions = ()
        self.arguments = ()

    def update(self, nodes=None, elemental_functions=None, arguments=None):
        self.nodes = as_tuple(nodes) or self.nodes
        self.elemental_functions = as_tuple(elemental_functions) or\
            self.elemental_functions
        self.arguments += as_tuple(arguments)

class Arg(object):

    """A DLE-produced argument."""

    from_Blocking = False

    def __init__(self, argument, value):
        self.argument = argument
        self.value = value

    def __repr__(self):
        return "DLE-GenericArg"

class BlockingArg(Arg):

    from_Blocking = True

    def __init__(self, blocked_dim, original_dim, value):
        """
        Represent an argument introduced in the kernel by Rewriter._loop_blocking.

        :param blocked_dim: The blocked :class:`Dimension`.
        :param original_dim: The original :class:`Dimension` corresponding
                             to ``blocked_dim``.
        :param value: A suggested value determined by the DLE.
        """
        super(BlockingArg, self).__init__(blocked_dim, value)
        self.original_dim = original_dim

    def __repr__(self):
        bsize = self.value if self.value else '<unused>'
        return "DLE-BlockingArg[%s,%s,%s]" % (self.argument, self.original_dim, bsize)

class Rewriter(object):

    triggers = {
        '_create_elemental_functions': ('split', 'advanced',),
        '_loop_blocking': ('blocking', 'advanced')
    }

    def __init__(self, nodes, params):
        self.nodes = nodes
        self.params = params

        self.timings = OrderedDict()

    def run(self, mode):
        state = State(self.nodes)

        self._analyze_and_decorate(state)

        self._create_elemental_functions(state, mode=mode)
        self._loop_blocking(state, mode=mode)

        self._summary(mode)

        return state

    def _analyze_and_decorate(self, state):
        """
        Analyze the Iteration/Expression trees in ``state.nodes`` and track
        useful information for the subsequent DLE's transformation steps.

        In particular, the presence of fully-parallel or "outermost-sequential
        inner-parallel" (OSIP) :class:`Iteration` trees is tracked. In an OSIP
        :class:`Iteration` tree, outermost :class:`Iteration` objects represent
        an inherently sequential dimension, whereas all inner :class:`Iteration`
        objects represent parallelizable dimensions.
        """

        nodes = state.nodes

        sections = FindSections().visit(nodes)
        trees = sections.keys()
        candidate = max(trees, key=lambda i: len(i))
        candidates = [i for i in trees if len(i) == len(candidate)]

        # The analysis below may return "false positives" (ie, absence of fully-
        # parallel or OSIP trees when this is actually false), but this should
        # never be the case in practice, given the targeted stencil codes.
        for tree in candidates:
            exprs = [e.stencil for e in sections[tree]]

            # "Prefetch" terminals to speed up the checks below
            terms = {e: tuple(terminals(e.rhs)) for e in exprs}

            # Does the Iteration index only appear in the outermost dimension ?
            has_parallel_dimension = True
            sample = None
            for k, v in terms.items():
                if v:
                    handle = v[0]
                    sample = sample or handle.indices[0]
                    if any(sample in i.indices[1:] or sample != i.indices[0] for i in v):
                        has_parallel_dimension = False
                        break
            if not has_parallel_dimension:
                continue

            # Is the Iteration tree fully-parallel or OSIP?
            is_OSIP = False
            for e in exprs:
                lhs = e.lhs
                if lhs.is_Symbol:
                    continue
                handle = [i for i in terms[e] if i.base.label == lhs.base.label]
                if any(lhs.indices[0] != i.indices[0] for i in handle):
                    is_OSIP = True
                    break

            # Track parallelism in the Iteration/Expression tree
            for i in tree[is_OSIP:]:
                args = i.args
                properties = as_tuple(args.pop('properties')) + ('parallel',)
                transformer = Transformer({i: Iteration(properties=properties, **args)})
                nodes = transformer.visit(nodes)

        state.update(nodes=nodes)

    @dle_transformation
    def _create_elemental_functions(self, state, **kwargs):
        """
        Move :class:`Iteration` sub-trees to separate functions.

        By default, inner iteration trees are moved. To move different types of
        :class:`Iteration`, one can provide a lambda function in ``kwargs['rule']``,
        taking as input an iterable of :class:`Iteration` and returning an iterable
        of :class:`Iteration` (eg, a subset, the whole iteration tree).
        """

        rule = kwargs.get('rule', lambda tree: tree[-1:])

        functions = []
        processed = []
        for i, node in enumerate(state.nodes):
            mapper = {}
            for j, tree in enumerate(retrieve_iteration_tree(node)):
                if len(tree) <= 1:
                    continue

                name = "f_%d_%d" % (i, j)

                candidate = rule(tree)
                leftover = tuple(k for k in tree if k not in candidate)

                args = FindSymbols().visit(candidate)
                args += [k.dim for k in leftover if k not in args and k.is_Closed]

                known = [k.name for k in args]
                known += [k.index for k in candidate]
                maybe_unknown = FindSymbols(mode='free-symbols').visit(candidate)
                args += [k for k in maybe_unknown if k.name not in known]

                call = []
                parameters = []
                for k in args:
                    if isinstance(k, Dimension):
                        call.append(k.ccode)
                        parameters.append(k)
                    elif isinstance(k, SymbolicData):
                        call.append("%s_vec" % k.name)
                        parameters.append(k)
                    else:
                        call.append(k.name)
                        parameters.append(ScalarData(name=k.name, dtype=np.int32))

                root = candidate[0]

                # Track info to transform the main tree
                call = '%s(%s)' % (name, ','.join(call))
                mapper[root] = Element(c.Statement(call))

                # Produce the new function
                functions.append(Function(name, root, 'void', parameters, ('static',)))

            # Transform the main tree
            processed.append(Transformer(mapper).visit(node))

        return {'nodes': processed, 'elemental_functions': functions}

    @dle_transformation
    def _loop_blocking(self, state, **kwargs):
        """
        Apply loop blocking to :class:`Iteartion` trees.

        By default, the blocked :class:`Iteration` objects and the block size are
        determined heuristically. The heuristic consists of searching the deepest
        Iteration/Expression tree and blocking all dimensions except:

            * The innermost (eg, to retain SIMD vectorization);
            * Those dimensions inducing loop-carried dependencies.

        The caller may take over the heuristic through ``kwargs['blocking']``,
        a dictionary indicating the block size of each blocked dimension. For
        example, for the :class:`Iteration` tree below: ::

            for i
              for j
                for k
                  ...

        one may pass in ``kwargs['blocking'] = {i: 4, j: 7}``, in which case the
        two outer loops would be blocked, and the resulting 2-dimensional block
        would be of size 4x7.
        """

        Region = namedtuple('Region', 'main leftover')

        is_InnerBlockable = self.params.get('blockinner',
                                            len(state.elemental_functions) > 0)

        dims = OrderedDict()
        processed = []
        for node in state.nodes:
            mapper = {}
            for tree in retrieve_iteration_tree(node):
                # Is the Iteration tree blockable ?
                iterations = [i for i in tree if 'parallel' in i.properties]
                iterations = iterations if is_InnerBlockable else iterations[:-1]
                if not iterations:
                    continue
                root = iterations[0]
                if not IsPerfectIteration().visit(root):
                    continue

                # Construct the blocked loop nest, as well as all necessary
                # remainder loops
                regions = {}
                blocked_iterations = []
                for i in iterations:
                    # Build Iteration over blocks
                    dim = dims.setdefault((i.dim, 'inter-block'),
                                          Dimension("%sb" % i.dim.name))

                    block_size = dim.symbolic_size
                    iter_size = i.dim.symbolic_size

                    start = i.limits[0] - i.offsets[0]
                    finish = iter_size - ((iter_size - i.offsets[1]) % block_size)
                    inter_block = Iteration([], dim, [start, finish, block_size])

                    # Build Iteration within a block
                    dim = dims.setdefault((i.dim, 'intra-block'),
                                          Dimension(i.dim.name))

                    start = inter_block.dim
                    finish = start + block_size
                    intra_block = Iteration([], dim, [start, finish, 1], i.index)

                    blocked_iterations.append((inter_block, intra_block))

                    # Build unitary-increment Iteration over the 'main' region
                    # (the one blocked); necessary to generate code iterating over
                    # non-blocked ("remainder") iterations.
                    start = inter_block.limits[0]
                    finish = inter_block.limits[1]
                    main = Iteration([], i.dim, [start, finish, 1], i.index)

                    # Build unitary-increment Iteration over the 'leftover' region:
                    # again as above, this may be necessary when the dimension size
                    # is not a multiple of the block size.
                    start = inter_block.limits[1]
                    finish = iter_size - i.offsets[1]
                    leftover = Iteration([], i.dim, [start, finish, 1], i.index)

                    regions[i] = Region(main, leftover)

                blocked_tree = list(flatten(zip(*blocked_iterations)))
                blocked = compose_nodes(blocked_tree + [iterations[-1].nodes])

                # Build remainder loops
                remainder_tree = []
                for n in range(len(iterations)):
                    for i in combinations(iterations, n + 1):
                        nodes = [v.leftover if k in i else v.main
                                 for k, v in regions.items()]
                        nodes += [iterations[-1].nodes]
                        remainder_tree.append(compose_nodes(nodes))

                # Will replace with blocked loop tree
                mapper[root] = List(body=[blocked] + remainder_tree)

            rebuilt = Transformer(mapper).visit(node)

            processed.append(rebuilt)

        # All blocked dimensions
        blocked = OrderedDict([(k, v) for k, v in dims.items() if k[1] == 'inter-block'])
        if not blocked:
            return {'nodes': processed}

        # Determine the block shape
        blockshape = self.params.get('blockshape')
        if not blockshape:
            # Use trivial heuristic for a suitable blockshape
            def heuristic(dim_size):
                ths = 8  # FIXME: This really needs to be improved
                return ths if dim_size > ths else 1
            blockshape = {k: heuristic for k in blocked.keys()}
        else:
            try:
                nitems, nrequired = len(blockshape), len(blocked)
                blockshape = {k: v for k, v in zip(blocked, blockshape)}
                if nitems > nrequired:
                    dle_warning("Provided 'blockshape' has more entries than "
                                "blocked loops; dropping entries ...")
                if nitems < nrequired:
                    dle_warning("Provided 'blockshape' has fewer entries than "
                                "blocked loops; dropping dimensions ...")
            except TypeError:
                blockshape = {blocked.keys()[0]: blockshape}
            blockshape.update({k: None for k in blocked.keys()
                               if k not in blockshape})

        # Track any additional arguments required to execute /state.nodes/
        arguments = [BlockingArg(v, k[0], blockshape[k]) for k, v in blocked.items()]

        return {'nodes': processed, 'arguments': arguments}

    def _summary(self, mode):
        """
        Print a summary of the DLE transformations
        """

        if mode.intersection({'blocking', 'advanced'}):
            steps = " --> ".join("(%s)" % i for i in self.timings.keys())
            elapsed = sum(self.timings.values())
            dle("%s [%.2f s]" % (steps, elapsed))
<commit_msg>dle: Change rewrite interface<commit_after>from __future__ import absolute_import

from collections import OrderedDict, Sequence, namedtuple
from itertools import combinations
from time import time

import numpy as np

import cgen as c

from devito.dimension import Dimension
from devito.dle.inspection import retrieve_iteration_tree
from devito.dle.manipulation import compose_nodes
from devito.dse import terminals
from devito.interfaces import ScalarData, SymbolicData
from devito.logger import dle, dle_warning
from devito.nodes import Element, Function, Iteration, List
from devito.tools import as_tuple, flatten
from devito.visitors import FindSections, FindSymbols, IsPerfectIteration, Transformer

def transform(node, mode='basic'):
    """
    Transform Iteration/Expression trees to generate highly optimized C code.

    :param node: The Iteration/Expression tree to be transformed, or an iterable
                 of Iteration/Expression trees.
    :param mode: Drive the tree transformation. ``mode`` can be a string indicating
                 a pre-established optimization sequence [S] or a tuple of individual
                 transformations [T]; in the latter case, the individual transformations
                 are composed. Available modes are: ::

                    * 'noop': Do nothing [S]
                    * 'advanced': Split elemental functions and apply loop blocking [S]
                    * 'blocking': Apply loop blocking [T]
                    * 'split': Identify and split elemental functions [T]

                 If ``mode`` is a tuple, the last entry may be used to provide optional
                 arguments to the DLE transformations. Accepted key-value pairs are: ::

                    * 'blockshape': A tuple representing the shape of a block created
                                    by loop blocking.
                    * 'blockinner': By default, loop blocking is not applied to the
                                    innermost dimension of an Iteration/Expression tree
                                    to maximize the chances of SIMD vectorization. To
                                    force the blocking of this loop, the ``blockinner``
                                    flag should be set to True.
    """

    if isinstance(node, Sequence):
        assert all(n.is_Node for n in node)
        node = list(node)
    elif node.is_Node:
        node = [node]
    else:
        raise ValueError("Got illegal node of type %s." % type(node))

    if not mode:
        return State(node)
    elif isinstance(mode, str):
        mode = list([mode])
    elif not isinstance(mode, Sequence):
        return State(node)
    else:
        try:
            mode = list(mode)
        except TypeError:
            dle_warning("Arg mode must be str or tuple (got %s)" % type(mode))
            return State(node)

    params = {}
    if isinstance(mode[-1], dict):
        params = mode.pop(-1)
    for k in params.keys():
        if k not in ('blockshape', 'blockinner'):
            dle_warning("Illegal DLE parameter '%s'" % str(k))
            params.pop(k)

    mode = set(mode)
    if mode.isdisjoint({'noop', 'blocking', 'split', 'advanced'}):
        dle_warning("Unknown transformer mode(s) %s" % str(mode))
        return State(node)
    else:
        return Rewriter(node, params).run(mode)

    return Rewriter(node).run(mode)

def dle_transformation(func):

    def wrapper(self, state, **kwargs):
        if kwargs['mode'].intersection(set(self.triggers[func.__name__])):
            tic = time()
            state.update(**func(self, state))
            toc = time()

            key = '%s%d' % (func.__name__, len(self.timings))
            self.timings[key] = toc - tic

    return wrapper

class State(object):

    """Represent the output of the DLE."""

    def __init__(self, nodes):
        self.nodes = as_tuple(nodes)

        self.elemental_functions = ()
        self.arguments = ()

    def update(self, nodes=None, elemental_functions=None, arguments=None):
        self.nodes = as_tuple(nodes) or self.nodes
        self.elemental_functions = as_tuple(elemental_functions) or\
            self.elemental_functions
        self.arguments += as_tuple(arguments)

class Arg(object):

    """A DLE-produced argument."""

    from_Blocking = False

    def __init__(self, argument, value):
        self.argument = argument
        self.value = value

    def __repr__(self):
        return "DLE-GenericArg"

class BlockingArg(Arg):

    from_Blocking = True

    def __init__(self, blocked_dim, original_dim, value):
        """
        Represent an argument introduced in the kernel by Rewriter._loop_blocking.

        :param blocked_dim: The blocked :class:`Dimension`.
        :param original_dim: The original :class:`Dimension` corresponding
                             to ``blocked_dim``.
        :param value: A suggested value determined by the DLE.
        """
        super(BlockingArg, self).__init__(blocked_dim, value)
        self.original_dim = original_dim

    def __repr__(self):
        bsize = self.value if self.value else '<unused>'
        return "DLE-BlockingArg[%s,%s,%s]" % (self.argument, self.original_dim, bsize)

class Rewriter(object):

    triggers = {
        '_create_elemental_functions': ('split', 'advanced',),
        '_loop_blocking': ('blocking', 'advanced')
    }

    def __init__(self, nodes, params):
        self.nodes = nodes
        self.params = params

        self.timings = OrderedDict()

    def run(self, mode):
        state = State(self.nodes)

        self._analyze_and_decorate(state)

        self._create_elemental_functions(state, mode=mode)
        self._loop_blocking(state, mode=mode)

        self._summary(mode)

        return state

    def _analyze_and_decorate(self, state):
        """
        Analyze the Iteration/Expression trees in ``state.nodes`` and track
        useful information for the subsequent DLE's transformation steps.

        In particular, the presence of fully-parallel or "outermost-sequential
        inner-parallel" (OSIP) :class:`Iteration` trees is tracked. In an OSIP
        :class:`Iteration` tree, outermost :class:`Iteration` objects represent
        an inherently sequential dimension, whereas all inner :class:`Iteration`
        objects represent parallelizable dimensions.
        """

        nodes = state.nodes

        sections = FindSections().visit(nodes)
        trees = sections.keys()
        candidate = max(trees, key=lambda i: len(i))
        candidates = [i for i in trees if len(i) == len(candidate)]

        # The analysis below may return "false positives" (ie, absence of fully-
        # parallel or OSIP trees when this is actually false), but this should
        # never be the case in practice, given the targeted stencil codes.
        for tree in candidates:
            exprs = [e.stencil for e in sections[tree]]

            # "Prefetch" terminals to speed up the checks below
            terms = {e: tuple(terminals(e.rhs)) for e in exprs}

            # Does the Iteration index only appear in the outermost dimension ?
            has_parallel_dimension = True
            sample = None
            for k, v in terms.items():
                if v:
                    handle = v[0]
                    sample = sample or handle.indices[0]
                    if any(sample in i.indices[1:] or sample != i.indices[0] for i in v):
                        has_parallel_dimension = False
                        break
            if not has_parallel_dimension:
                continue

            # Is the Iteration tree fully-parallel or OSIP?
            is_OSIP = False
            for e in exprs:
                lhs = e.lhs
                if lhs.is_Symbol:
                    continue
                handle = [i for i in terms[e] if i.base.label == lhs.base.label]
                if any(lhs.indices[0] != i.indices[0] for i in handle):
                    is_OSIP = True
                    break

            # Track parallelism in the Iteration/Expression tree
            for i in tree[is_OSIP:]:
                args = i.args
                properties = as_tuple(args.pop('properties')) + ('parallel',)
                transformer = Transformer({i: Iteration(properties=properties, **args)})
                nodes = transformer.visit(nodes)

        state.update(nodes=nodes)

    @dle_transformation
    def _create_elemental_functions(self, state, **kwargs):
        """
        Move :class:`Iteration` sub-trees to separate functions.

        By default, inner iteration trees are moved. To move different types of
        :class:`Iteration`, one can provide a lambda function in ``kwargs['rule']``,
        taking as input an iterable of :class:`Iteration` and returning an iterable
        of :class:`Iteration` (eg, a subset, the whole iteration tree).
        """

        rule = kwargs.get('rule', lambda tree: tree[-1:])

        functions = []
        processed = []
        for i, node in enumerate(state.nodes):
            mapper = {}
            for j, tree in enumerate(retrieve_iteration_tree(node)):
                if len(tree) <= 1:
                    continue

                name = "f_%d_%d" % (i, j)

                candidate = rule(tree)
                leftover = tuple(k for k in tree if k not in candidate)

                args = FindSymbols().visit(candidate)
                args += [k.dim for k in leftover if k not in args and k.is_Closed]

                known = [k.name for k in args]
                known += [k.index for k in candidate]
                maybe_unknown = FindSymbols(mode='free-symbols').visit(candidate)
                args += [k for k in maybe_unknown if k.name not in known]

                call = []
                parameters = []
                for k in args:
                    if isinstance(k, Dimension):
                        call.append(k.ccode)
                        parameters.append(k)
                    elif isinstance(k, SymbolicData):
                        call.append("%s_vec" % k.name)
                        parameters.append(k)
                    else:
                        call.append(k.name)
                        parameters.append(ScalarData(name=k.name, dtype=np.int32))

                root = candidate[0]

                # Track info to transform the main tree
                call = '%s(%s)' % (name, ','.join(call))
                mapper[root] = Element(c.Statement(call))

                # Produce the new function
                functions.append(Function(name, root, 'void', parameters, ('static',)))

            # Transform the main tree
            processed.append(Transformer(mapper).visit(node))

        return {'nodes': processed, 'elemental_functions': functions}

    @dle_transformation
    def _loop_blocking(self, state, **kwargs):
        """
        Apply loop blocking to :class:`Iteartion` trees.

        By default, the blocked :class:`Iteration` objects and the block size are
        determined heuristically. The heuristic consists of searching the deepest
        Iteration/Expression tree and blocking all dimensions except:

            * The innermost (eg, to retain SIMD vectorization);
            * Those dimensions inducing loop-carried dependencies.

        The caller may take over the heuristic through ``kwargs['blocking']``,
        a dictionary indicating the block size of each blocked dimension. For
        example, for the :class:`Iteration` tree below: ::

            for i
              for j
                for k
                  ...

        one may pass in ``kwargs['blocking'] = {i: 4, j: 7}``, in which case the
        two outer loops would be blocked, and the resulting 2-dimensional block
        would be of size 4x7.
        """

        Region = namedtuple('Region', 'main leftover')

        is_InnerBlockable = self.params.get('blockinner',
                                            len(state.elemental_functions) > 0)

        dims = OrderedDict()
        processed = []
        for node in state.nodes:
            mapper = {}
            for tree in retrieve_iteration_tree(node):
                # Is the Iteration tree blockable ?
                iterations = [i for i in tree if 'parallel' in i.properties]
                iterations = iterations if is_InnerBlockable else iterations[:-1]
                if not iterations:
                    continue
                root = iterations[0]
                if not IsPerfectIteration().visit(root):
                    continue

                # Construct the blocked loop nest, as well as all necessary
                # remainder loops
                regions = {}
                blocked_iterations = []
                for i in iterations:
                    # Build Iteration over blocks
                    dim = dims.setdefault((i.dim, 'inter-block'),
                                          Dimension("%sb" % i.dim.name))

                    block_size = dim.symbolic_size
                    iter_size = i.dim.symbolic_size

                    start = i.limits[0] - i.offsets[0]
                    finish = iter_size - ((iter_size - i.offsets[1]) % block_size)
                    inter_block = Iteration([], dim, [start, finish, block_size])

                    # Build Iteration within a block
                    dim = dims.setdefault((i.dim, 'intra-block'),
                                          Dimension(i.dim.name))

                    start = inter_block.dim
                    finish = start + block_size
                    intra_block = Iteration([], dim, [start, finish, 1], i.index)

                    blocked_iterations.append((inter_block, intra_block))

                    # Build unitary-increment Iteration over the 'main' region
                    # (the one blocked); necessary to generate code iterating over
                    # non-blocked ("remainder") iterations.
                    start = inter_block.limits[0]
                    finish = inter_block.limits[1]
                    main = Iteration([], i.dim, [start, finish, 1], i.index)

                    # Build unitary-increment Iteration over the 'leftover' region:
                    # again as above, this may be necessary when the dimension size
                    # is not a multiple of the block size.
                    start = inter_block.limits[1]
                    finish = iter_size - i.offsets[1]
                    leftover = Iteration([], i.dim, [start, finish, 1], i.index)

                    regions[i] = Region(main, leftover)

                blocked_tree = list(flatten(zip(*blocked_iterations)))
                blocked = compose_nodes(blocked_tree + [iterations[-1].nodes])

                # Build remainder loops
                remainder_tree = []
                for n in range(len(iterations)):
                    for i in combinations(iterations, n + 1):
                        nodes = [v.leftover if k in i else v.main
                                 for k, v in regions.items()]
                        nodes += [iterations[-1].nodes]
                        remainder_tree.append(compose_nodes(nodes))

                # Will replace with blocked loop tree
                mapper[root] = List(body=[blocked] + remainder_tree)

            rebuilt = Transformer(mapper).visit(node)

            processed.append(rebuilt)

        # All blocked dimensions
        blocked = OrderedDict([(k, v) for k, v in dims.items() if k[1] == 'inter-block'])
        if not blocked:
            return {'nodes': processed}

        # Determine the block shape
        blockshape = self.params.get('blockshape')
        if not blockshape:
            # Use trivial heuristic for a suitable blockshape
            def heuristic(dim_size):
                ths = 8  # FIXME: This really needs to be improved
                return ths if dim_size > ths else 1
            blockshape = {k: heuristic for k in blocked.keys()}
        else:
            try:
                nitems, nrequired = len(blockshape), len(blocked)
                blockshape = {k: v for k, v in zip(blocked, blockshape)}
                if nitems > nrequired:
                    dle_warning("Provided 'blockshape' has more entries than "
                                "blocked loops; dropping entries ...")
                if nitems < nrequired:
                    dle_warning("Provided 'blockshape' has fewer entries than "
                                "blocked loops; dropping dimensions ...")
            except TypeError:
                blockshape = {blocked.keys()[0]: blockshape}
            blockshape.update({k: None for k in blocked.keys()
                               if k not in blockshape})

        # Track any additional arguments required to execute /state.nodes/
        arguments = [BlockingArg(v, k[0], blockshape[k]) for k, v in blocked.items()]

        return {'nodes': processed, 'arguments': arguments}

    def _summary(self, mode):
        """
        Print a summary of the DLE transformations
        """

        if mode.intersection({'blocking', 'advanced'}):
            steps = " --> ".join("(%s)" % i for i in self.timings.keys())
            elapsed = sum(self.timings.values())
            dle("%s [%.2f s]" % (steps, elapsed))
<|endoftext|>