<commit_before>import json
import xml.etree.cElementTree as CET
import StringIO
import string
import ResponseInterpreter

class CasparTalker:
    # CasparTalker is kind of a translation layer between the interface and Casper's
    # control protocol (currently AMCP).
    # It will build the correct AMCP string for a given command, tell the
    # CasparServer to send it, and handles the response.

    def __init__(self, server=None):
        # TODO: Check we're using the right version of AMCP. This is currently 2.1

        # fixme: this is more a placeholder - make this actually work with the response of INFO PATHS
        # if server:
        # paths_dec = []
        # for p in self.info_paths(server):
        # TODO: json decode this info
        pass

    def send_command_to_caspar(self, server, amcp_command):
        print "Sending command:", amcp_command
        if not amcp_command.endswith("\r\n"): amcp_command += "\r\n"
        server.send_command(amcp_command)

        response = server.read_until("\r\n")

        # ResponseInterpreter lets us know how to proceed - Caspar's way of sending information
        # is a bit vague.
        to_do = ResponseInterpreter.interpret_response(response)
        if to_do[1]:
            return server.read_until(to_do[2])
        else:
            return None

    # Query commands - return info about various things

    def tls(self, server):
        # Lists all template files in the templates folder.
        # Use the command INFO PATHS to get the path to the templates folder.

        amcp_string = "TLS"
        templates_response = self.send_command_to_caspar(server, amcp_string)
        templates = []

        # The template list is returned in the following fashion (each line is an array entry):
        # "RELATIVE-PATH/TEMPLATE-NAME" SIZE-IN-BYTES TIMESTAMP
        # We'll strip the first quote and everything after (and including) the last quote

        for t in templates_response:
            if not t[0] == '"':
                break
            templates.append(t.split('"')[1])

        return templates

    def version(self, server, component=None):
        # VERSION {[component:string]}
        # Returns the version of specified component.

        if component:
            amcp_string = "VERSION {0}".format(component)
        else:
            amcp_string = "VERSION"

        response = self.send_command_to_caspar(server, amcp_string)

        if response:
            return response[0]
        else:
            return None

    def info(self, server, channel=None, layer=None):
        # INFO [video_channel:int]{-[layer:int]}
        # Get information about a channel or a specific layer on a channel.
        # If layer is omitted information about the whole channel is returned.
        # If both video_channel and layer are omitted, retrieves a list of the available
        # channels

        if channel:
            if layer:
                amcp_string = "INFO {video_channel}-{layer}".format(video_channel=channel,
                                                                    layer=layer)
            else:
                amcp_string = "INFO {video_channel}".format(video_channel=channel)
        else:
            amcp_string = "INFO"

        response = self.send_command_to_caspar(server, amcp_string)

        if response:
            return response[0]
        else:
            return None

    def info_template(self, server, template):
        # INFO TEMPLATE [template:string]
        # Gets information about the specified template.
        # TODO: implement INFO TEMPLATE command - though this can't really be done until CCG 2.1 is released

        amcp_string = "INFO TEMPLATE {template}".format(template=template)

        response = self.send_command_to_caspar(server, amcp_string)
        for r in response:
            print r

        raise NotImplementedError

    def info_config(self, server):
        # INFO CONFIG
        # Gets the contents of the configuration used.

        amcp_string = "INFO CONFIG"

        response = self.send_command_to_caspar(server, amcp_string)
        response = StringIO.StringIO(string.join(response, ""))

        config = {}

        for event, elem in CET.iterparse(response):

            if "path" in elem.tag:
                print "Skipping <paths> - use INFO PATHS instead"
                elem.clear()
            else:
                config[elem.tag] = elem.text
                elem.clear()

        # Just as a point, this doesn't really return anything useful yet, but we'll send something back anyway.

        return config

    def info_paths(self, server):
        # INFO PATHS
        # Gets information about the paths used

        amcp_string = "INFO PATHS"
        response = self.send_command_to_caspar(server, amcp_string)

        if not response: return None

        paths = {}
        response = StringIO.StringIO(string.join(response, ""))

        for event, elem in CET.iterparse(response):
            if "-path" in elem.tag:
                # Huzzah, we've found a path!
                print "Found", elem.tag, elem.text
                paths[elem.tag.split("-")[0]] = elem.text
                elem.clear()

        return paths

    def info_system(self, server):
        # INFO SYSTEM
        # Gets system information like OS, CPU and library version numbers.
        # TODO: implement INFO SYSTEM command

        #amcp_string = "INFO SYSTEM"

        #response = self.send_command_to_caspar(server, amcp_string)
        #for r in response: print r
        #response = StringIO.StringIO(string.join(response, ""))

        #for event, elem in CET.iterparse(response):
        #    if elem.tag == "paths":
        #        print "Skipping <paths> - use INFO PATHS instead"
        #        elem.clear()
        #    else: print CET.dump(elem)

        raise NotImplementedError

    def info_server(self, server):
        # INFO SERVER
        # Gets detailed information about all channels.
        # TODO: implement INFO SERVER command

        raise NotImplementedError

    def bye(self, server):
        # BYE
        # Disconnects from the server if connected remotely,
        # if interacting directly with the console on the machine Caspar is running on
        # then this will achieve the same as the KILL command.
        # TODO: implement BYE

        amcp_string = "BYE"

        self.send_command_to_caspar(server, amcp_string)

        return True

    def kill(self, server):
        # KILL
        # Shuts the server down.
        # TODO: implement KILL command

        raise NotImplementedError

    # Data commands - create and manipulate datasets

    def data_store(self, server, name, data):
        # DATA STORE [name:string] [data:string]
        # Stores the dataset data under the name name.
        # Directories will be created if they do not exist.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "DATA STORE {name} {data}".format(name=name, data=data)

        self.send_command_to_caspar(server, amcp_string)

    def data_retrieve(self, server, name):
        # DATA RETRIEVE [name:string]
        # Returns the data saved under the name name.

        amcp_string = "DATA RETRIEVE {name}".format(name=name)

        data = self.send_command_to_caspar(server, amcp_string)
        return data

    def data_list(self, server):
        # DATA LIST
        # Returns a list of all stored datasets.

        amcp_string = "DATA LIST"
        data = self.send_command_to_caspar(server, amcp_string)
        return data

    def data_remove(self, server, name):
        # DATA REMOVE [name:string]
        # Removes the dataset saved under the name name.
        # fixme - a bug with CCG 2.0 means this doens't work, as CCG returns a code 200, rather than 201.
        # This is fixed in CCG 2.1

        amcp_string = "DATA REMOVE {name}".format(name=name)
        self.send_command_to_caspar(server, amcp_string)

    # CG Commands - manipulate Flash templates in Caspar
    def cg_add(self, server, template, channel=1, layer=10, cg_layer=0, play_on_load=0, data=None):
        # CG [video_channel:int]{-[layer:int]|-9999} ADD [cg_layer:int] [template:string] [play-on-load:0,1] {[data]}
        # Prepares a template for displaying.
        # It won't show until you call CG PLAY (unless you supply the play-on-load flag, 1 for true).
        # Data is either inline XML or a reference to a saved dataset.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "CG {video_channel}-{layer} ADD {cg_layer} {template} {play_on_load} {data}".format(
            video_channel=channel, layer=layer, cg_layer=cg_layer, template=template, play_on_load=play_on_load,
            data=data)

        self.send_command_to_caspar(server, amcp_string)

    def cg_play(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} PLAY [cg_layer:int]
        # Plays and displays the template in the specified layer.

        amcp_string = "CG {video_channel}-{layer} PLAY {cg_layer}".format(video_channel=channel,
                                                                          layer=layer, cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_stop(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} STOP [cg_layer:int]
        # Stops and removes the template from the specified layer.
        # This is different from REMOVE in that the template gets a chance to
        # animate out when it is stopped.

        amcp_string = "CG {video_channel}-{layer} STOP [cg_layer]".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_next(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} NEXT [cg_layer:int]
        # Triggers a "continue" in the template on the specified layer.
        # This is used to control animations that has multiple discreet steps.

        amcp_string = "CG {video_channel}-{layer} NEXT {cg_layer}".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_remove(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} REMOVE [cg_layer:int]
        # Removes the template from the specified layer.

        amcp_string = "CG {video_channel}-{layer} REMOVE {cg_layer}".format(video_channel=channel, layer=layer,
                                                                            cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_clear(self, server, channel=1, layer=10):
        # CG [video_channel:int]{-[layer:int]|-9999} CLEAR
        # Removes all templates on a video layer. The entire cg producer will be removed.

        amcp_string = "CG {video_channel}-{layer} CLEAR".format(video_channel=channel, layer=layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_update(self, server, channel=1, layer=10, data=None):
        # CG [video_channel:int]{-[layer:int]|-9999} UPDATE [cg_layer:int] [data:string]
        # Sends new data to the template on specified layer.
        # Data is either inline XML or a reference to a saved dataset.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "CG {video_channel}-{layer} UPDATE {cg_layer} {data}".format(video_channel=channel,
                                                                                layer=layer, data=data)

        self.send_command_to_caspar(server, amcp_string)

    def cg_invoke(self, server, method, channel=1, layer=0, cg_layer=1):
        # CG [video_channel:int]{-[layer:int]|-9999} INVOKE [cg_layer:int] [method:string]
        # Invokes the given method on the template on the specified layer.
        # Can be used to jump the playhead to a specific label.

        amcp_string = "CG {video_channel}-{layer} INVOKE {cg_layer} {method}".format(video_channel=channel, layer=layer,
                                                                                cg_layer=cg_layer, method=method)

        self.send_command_to_caspar(server, amcp_string)

    def cg_info(self, server, channel=1, layer=10, cg_layer=1):
        # CG [video_channel:int]{-[layer:int]|-9999} INFO {[cg_layer:int]}
        # Retrieves information about the template on the specified layer.
        # If cg_layer is not given, information about the template host is given instead.

        amcp_string = "CG {video_channel}-{layer} INFO {cg_layer}".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        print self.send_command_to_caspar(server, amcp_string)
<commit_msg>ubercarrot-7 The KILL AMCP 2.1 command has not been implemented<commit_after>import json
import xml.etree.cElementTree as CET
import StringIO
import string
import ResponseInterpreter

class CasparTalker:
    # CasparTalker is kind of a translation layer between the interface and Casper's
    # control protocol (currently AMCP).
    # It will build the correct AMCP string for a given command, tell the
    # CasparServer to send it, and handles the response.

    def __init__(self, server=None):
        # TODO: Check we're using the right version of AMCP. This is currently 2.1

        # fixme: this is more a placeholder - make this actually work with the response of INFO PATHS
        # if server:
        # paths_dec = []
        # for p in self.info_paths(server):
        # TODO: json decode this info
        pass

    def send_command_to_caspar(self, server, amcp_command):
        print "Sending command:", amcp_command
        if not amcp_command.endswith("\r\n"): amcp_command += "\r\n"
        server.send_command(amcp_command)

        response = server.read_until("\r\n")

        # ResponseInterpreter lets us know how to proceed - Caspar's way of sending information
        # is a bit vague.
        to_do = ResponseInterpreter.interpret_response(response)
        if to_do[1]:
            return server.read_until(to_do[2])
        else:
            return None

    # Query commands - return info about various things

    def tls(self, server):
        # Lists all template files in the templates folder.
        # Use the command INFO PATHS to get the path to the templates folder.

        amcp_string = "TLS"
        templates_response = self.send_command_to_caspar(server, amcp_string)
        templates = []

        # The template list is returned in the following fashion (each line is an array entry):
        # "RELATIVE-PATH/TEMPLATE-NAME" SIZE-IN-BYTES TIMESTAMP
        # We'll strip the first quote and everything after (and including) the last quote

        for t in templates_response:
            if not t[0] == '"':
                break
            templates.append(t.split('"')[1])

        return templates

    def version(self, server, component=None):
        # VERSION {[component:string]}
        # Returns the version of specified component.

        if component:
            amcp_string = "VERSION {0}".format(component)
        else:
            amcp_string = "VERSION"

        response = self.send_command_to_caspar(server, amcp_string)

        if response:
            return response[0]
        else:
            return None

    def info(self, server, channel=None, layer=None):
        # INFO [video_channel:int]{-[layer:int]}
        # Get information about a channel or a specific layer on a channel.
        # If layer is omitted information about the whole channel is returned.
        # If both video_channel and layer are omitted, retrieves a list of the available
        # channels

        if channel:
            if layer:
                amcp_string = "INFO {video_channel}-{layer}".format(video_channel=channel,
                                                                    layer=layer)
            else:
                amcp_string = "INFO {video_channel}".format(video_channel=channel)
        else:
            amcp_string = "INFO"

        response = self.send_command_to_caspar(server, amcp_string)

        if response:
            return response[0]
        else:
            return None

    def info_template(self, server, template):
        # INFO TEMPLATE [template:string]
        # Gets information about the specified template.
        # TODO: implement INFO TEMPLATE command - though this can't really be done until CCG 2.1 is released

        amcp_string = "INFO TEMPLATE {template}".format(template=template)

        response = self.send_command_to_caspar(server, amcp_string)
        for r in response:
            print r

        raise NotImplementedError

    def info_config(self, server):
        # INFO CONFIG
        # Gets the contents of the configuration used.

        amcp_string = "INFO CONFIG"

        response = self.send_command_to_caspar(server, amcp_string)
        response = StringIO.StringIO(string.join(response, ""))

        config = {}

        for event, elem in CET.iterparse(response):

            if "path" in elem.tag:
                print "Skipping <paths> - use INFO PATHS instead"
                elem.clear()
            else:
                config[elem.tag] = elem.text
                elem.clear()

        # Just as a point, this doesn't really return anything useful yet, but we'll send something back anyway.

        return config

    def info_paths(self, server):
        # INFO PATHS
        # Gets information about the paths used

        amcp_string = "INFO PATHS"
        response = self.send_command_to_caspar(server, amcp_string)

        if not response: return None

        paths = {}
        response = StringIO.StringIO(string.join(response, ""))

        for event, elem in CET.iterparse(response):
            if "-path" in elem.tag:
                # Huzzah, we've found a path!
                print "Found", elem.tag, elem.text
                paths[elem.tag.split("-")[0]] = elem.text
                elem.clear()

        return paths

    def info_system(self, server):
        # INFO SYSTEM
        # Gets system information like OS, CPU and library version numbers.
        # TODO: implement INFO SYSTEM command

        #amcp_string = "INFO SYSTEM"

        #response = self.send_command_to_caspar(server, amcp_string)
        #for r in response: print r
        #response = StringIO.StringIO(string.join(response, ""))

        #for event, elem in CET.iterparse(response):
        #    if elem.tag == "paths":
        #        print "Skipping <paths> - use INFO PATHS instead"
        #        elem.clear()
        #    else: print CET.dump(elem)

        raise NotImplementedError

    def info_server(self, server):
        # INFO SERVER
        # Gets detailed information about all channels.
        # TODO: implement INFO SERVER command

        raise NotImplementedError

    def bye(self, server):
        # BYE
        # Disconnects from the server if connected remotely,
        # if interacting directly with the console on the machine Caspar is running on
        # then this will achieve the same as the KILL command.

        amcp_string = "BYE"

        self.send_command_to_caspar(server, amcp_string)

        return True

    def kill(self, server):
        # KILL
        # Shuts the server down.

        amcp_string = "KILL"

        self.send_command_to_caspar(server, amcp_string)

        return True

    # Data commands - create and manipulate datasets

    def data_store(self, server, name, data):
        # DATA STORE [name:string] [data:string]
        # Stores the dataset data under the name name.
        # Directories will be created if they do not exist.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "DATA STORE {name} {data}".format(name=name, data=data)

        self.send_command_to_caspar(server, amcp_string)

    def data_retrieve(self, server, name):
        # DATA RETRIEVE [name:string]
        # Returns the data saved under the name name.

        amcp_string = "DATA RETRIEVE {name}".format(name=name)

        data = self.send_command_to_caspar(server, amcp_string)
        return data

    def data_list(self, server):
        # DATA LIST
        # Returns a list of all stored datasets.

        amcp_string = "DATA LIST"
        data = self.send_command_to_caspar(server, amcp_string)
        return data

    def data_remove(self, server, name):
        # DATA REMOVE [name:string]
        # Removes the dataset saved under the name name.
        # fixme - a bug with CCG 2.0 means this doens't work, as CCG returns a code 200, rather than 201.
        # This is fixed in CCG 2.1

        amcp_string = "DATA REMOVE {name}".format(name=name)
        self.send_command_to_caspar(server, amcp_string)

    # CG Commands - manipulate Flash templates in Caspar
    def cg_add(self, server, template, channel=1, layer=10, cg_layer=0, play_on_load=0, data=None):
        # CG [video_channel:int]{-[layer:int]|-9999} ADD [cg_layer:int] [template:string] [play-on-load:0,1] {[data]}
        # Prepares a template for displaying.
        # It won't show until you call CG PLAY (unless you supply the play-on-load flag, 1 for true).
        # Data is either inline XML or a reference to a saved dataset.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "CG {video_channel}-{layer} ADD {cg_layer} {template} {play_on_load} {data}".format(
            video_channel=channel, layer=layer, cg_layer=cg_layer, template=template, play_on_load=play_on_load,
            data=data)

        self.send_command_to_caspar(server, amcp_string)

    def cg_play(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} PLAY [cg_layer:int]
        # Plays and displays the template in the specified layer.

        amcp_string = "CG {video_channel}-{layer} PLAY {cg_layer}".format(video_channel=channel,
                                                                          layer=layer, cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_stop(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} STOP [cg_layer:int]
        # Stops and removes the template from the specified layer.
        # This is different from REMOVE in that the template gets a chance to
        # animate out when it is stopped.

        amcp_string = "CG {video_channel}-{layer} STOP [cg_layer]".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_next(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} NEXT [cg_layer:int]
        # Triggers a "continue" in the template on the specified layer.
        # This is used to control animations that has multiple discreet steps.

        amcp_string = "CG {video_channel}-{layer} NEXT {cg_layer}".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_remove(self, server, channel=1, layer=10, cg_layer=0):
        # CG [video_channel:int]{-[layer:int]|-9999} REMOVE [cg_layer:int]
        # Removes the template from the specified layer.

        amcp_string = "CG {video_channel}-{layer} REMOVE {cg_layer}".format(video_channel=channel, layer=layer,
                                                                            cg_layer=cg_layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_clear(self, server, channel=1, layer=10):
        # CG [video_channel:int]{-[layer:int]|-9999} CLEAR
        # Removes all templates on a video layer. The entire cg producer will be removed.

        amcp_string = "CG {video_channel}-{layer} CLEAR".format(video_channel=channel, layer=layer)

        self.send_command_to_caspar(server, amcp_string)

    def cg_update(self, server, channel=1, layer=10, data=None):
        # CG [video_channel:int]{-[layer:int]|-9999} UPDATE [cg_layer:int] [data:string]
        # Sends new data to the template on specified layer.
        # Data is either inline XML or a reference to a saved dataset.

        data = json.dumps(data)  # Escape quotes, etc.
        amcp_string = "CG {video_channel}-{layer} UPDATE {cg_layer} {data}".format(video_channel=channel,
                                                                                layer=layer, data=data)

        self.send_command_to_caspar(server, amcp_string)

    def cg_invoke(self, server, method, channel=1, layer=0, cg_layer=1):
        # CG [video_channel:int]{-[layer:int]|-9999} INVOKE [cg_layer:int] [method:string]
        # Invokes the given method on the template on the specified layer.
        # Can be used to jump the playhead to a specific label.

        amcp_string = "CG {video_channel}-{layer} INVOKE {cg_layer} {method}".format(video_channel=channel, layer=layer,
                                                                                cg_layer=cg_layer, method=method)

        self.send_command_to_caspar(server, amcp_string)

    def cg_info(self, server, channel=1, layer=10, cg_layer=1):
        # CG [video_channel:int]{-[layer:int]|-9999} INFO {[cg_layer:int]}
        # Retrieves information about the template on the specified layer.
        # If cg_layer is not given, information about the template host is given instead.

        amcp_string = "CG {video_channel}-{layer} INFO {cg_layer}".format(video_channel=channel, layer=layer,
                                                                          cg_layer=cg_layer)

        print self.send_command_to_caspar(server, amcp_string)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import datetime
import logging
import psycopg2
import threading

from collections import defaultdict
from email.utils import formataddr

from odoo import _, api, fields, models
from odoo import tools
from odoo.addons.base.ir.ir_mail_server import MailDeliveryException
from odoo.tools.safe_eval import safe_eval

_logger = logging.getLogger(__name__)

class MailMail(models.Model):
    """ Model holding RFC2822 email messages to send. This model also provides
        facilities to queue and send new email messages.  """
    _name = 'mail.mail'
    _description = 'Outgoing Mails'
    _inherits = {'mail.message': 'mail_message_id'}
    _order = 'id desc'
    _rec_name = 'subject'

    # content
    mail_message_id = fields.Many2one('mail.message', 'Message', required=True, ondelete='cascade', index=True, auto_join=True)
    body_html = fields.Text('Rich-text Contents', help="Rich-text/HTML message")
    references = fields.Text('References', help='Message references, such as identifiers of previous messages', readonly=1)
    headers = fields.Text('Headers', copy=False)
    # Auto-detected based on create() - if 'mail_message_id' was passed then this mail is a notification
    # and during unlink() we will not cascade delete the parent and its attachments
    notification = fields.Boolean('Is Notification', help='Mail has been created to notify people of an existing mail.message')
    # recipients
    email_to = fields.Text('To', help='Message recipients (emails)')
    email_cc = fields.Char('Cc', help='Carbon copy message recipients')
    recipient_ids = fields.Many2many('res.partner', string='To (Partners)')
    # process
    state = fields.Selection([
        ('outgoing', 'Outgoing'),
        ('sent', 'Sent'),
        ('received', 'Received'),
        ('exception', 'Delivery Failed'),
        ('cancel', 'Cancelled'),
    ], 'Status', readonly=True, copy=False, default='outgoing')
    auto_delete = fields.Boolean(
        'Auto Delete',
        help="Permanently delete this email after sending it, to save space")
    failure_reason = fields.Text(
        'Failure Reason', readonly=1,
        help="Failure reason. This is usually the exception thrown by the email server, stored to ease the debugging of mailing issues.")
    scheduled_date = fields.Char('Scheduled Send Date',
        help="If set, the queue manager will send the email after the date. If not set, the email will be send as soon as possible.")

    @api.model
    def create(self, values):
        # notification field: if not set, set if mail comes from an existing mail.message
        if 'notification' not in values and values.get('mail_message_id'):
            values['notification'] = True
        if not values.get('mail_message_id'):
            self = self.with_context(message_create_from_mail_mail=True)
        return super(MailMail, self).create(values)

    @api.multi
    def unlink(self):
        # cascade-delete the parent message for all mails that are not created for a notification
        to_cascade = self.search([('notification', '=', False), ('id', 'in', self.ids)]).mapped('mail_message_id')
        res = super(MailMail, self).unlink()
        to_cascade.unlink()
        return res

    @api.model
    def default_get(self, fields):
        # protection for `default_type` values leaking from menu action context (e.g. for invoices)
        # To remove when automatic context propagation is removed in web client
        if self._context.get('default_type') not in type(self).message_type.base_field.selection:
            self = self.with_context(dict(self._context, default_type=None))
        return super(MailMail, self).default_get(fields)

    @api.multi
    def mark_outgoing(self):
        return self.write({'state': 'outgoing'})

    @api.multi
    def cancel(self):
        return self.write({'state': 'cancel'})

    @api.model
    def process_email_queue(self, ids=None):
        """Send immediately queued messages, committing after each
           message is sent - this is not transactional and should
           not be called during another transaction!

           :param list ids: optional list of emails ids to send. If passed
                            no search is performed, and these ids are used
                            instead.
           :param dict context: if a 'filters' key is present in context,
                                this value will be used as an additional
                                filter to further restrict the outgoing
                                messages to send (by default all 'outgoing'
                                messages are sent).
        """
        if not self.ids:
            filters = ['&',
                       ('state', '=', 'outgoing'),
                       '|',
                       ('scheduled_date', '<', datetime.datetime.now()),
                       ('scheduled_date', '=', False)]
            if 'filters' in self._context:
                filters.extend(self._context['filters'])
            # TODO: make limit configurable
            ids = self.search(filters, limit=10000).ids
        res = None
        try:
            # auto-commit except in testing mode
            auto_commit = not getattr(threading.currentThread(), 'testing', False)
            res = self.browse(ids).send(auto_commit=auto_commit)
        except Exception:
            _logger.exception("Failed processing mail queue")
        return res

    @api.multi
    def _postprocess_sent_message(self, mail_sent=True):
        """Perform any post-processing necessary after sending ``mail``
        successfully, including deleting it completely along with its
        attachment if the ``auto_delete`` flag of the mail was set.
        Overridden by subclasses for extra post-processing behaviors.

        :param browse_record mail: the mail that was just sent
        :return: True
        """
        notif_emails = self.filtered(lambda email: email.notification)
        if notif_emails:
            notifications = self.env['mail.notification'].search([
                ('mail_message_id', 'in', notif_emails.mapped('mail_message_id').ids),
                ('is_email', '=', True)])
            if mail_sent:
                notifications.write({
                    'email_status': 'sent',
                })
            else:
                notifications.write({
                    'email_status': 'exception',
                })
        if mail_sent:
            self.sudo().filtered(lambda self: self.auto_delete).unlink()
        return True

    # ------------------------------------------------------
    # mail_mail formatting, tools and send mechanism
    # ------------------------------------------------------

    @api.multi
    def send_get_mail_body(self, partner=None):
        """Return a specific ir_email body. The main purpose of this method
        is to be inherited to add custom content depending on some module."""
        self.ensure_one()
        body = self.body_html or ''
        return body

    @api.multi
    def send_get_mail_to(self, partner=None):
        """Forge the email_to with the following heuristic:
          - if 'partner', recipient specific (Partner Name <email>)
          - else fallback on mail.email_to splitting """
        self.ensure_one()
        if partner:
            email_to = [formataddr((partner.name, partner.email))]
        else:
            email_to = tools.email_split_and_format(self.email_to)
        return email_to

    @api.multi
    def send_get_email_dict(self, partner=None):
        """Return a dictionary for specific email values, depending on a
        partner, or generic to the whole recipients given by mail.email_to.

            :param browse_record mail: mail.mail browse_record
            :param browse_record partner: specific recipient partner
        """
        self.ensure_one()
        body = self.send_get_mail_body(partner=partner)
        body_alternative = tools.html2plaintext(body)
        res = {
            'body': body,
            'body_alternative': body_alternative,
            'email_to': self.send_get_mail_to(partner=partner),
        }
        return res

    @api.multi
    def _split_by_server(self):
        """Returns an iterator of pairs `(mail_server_id, record_ids)` for current recordset.

        The same `mail_server_id` may repeat in order to limit batch size according to
        the `mail.session.batch.size` system parameter.
        """
        groups = defaultdict(list)
        # Turn prefetch OFF to avoid MemoryError on very large mail queues, we only care
        # about the mail server ids in this case.
        for mail in self.with_context(prefetch_fields=False):
            groups[mail.mail_server_id.id].append(mail.id)
        sys_params = self.env['ir.config_parameter'].sudo()
        batch_size = int(sys_params.get_param('mail.session.batch.size', 1000))
        for server_id, record_ids in groups.iteritems():
            for mail_batch in tools.split_every(batch_size, record_ids):
                yield server_id, mail_batch

    @api.multi
    def send(self, auto_commit=False, raise_exception=False):
        """ Sends the selected emails immediately, ignoring their current
            state (mails that have already been sent should not be passed
            unless they should actually be re-sent).
            Emails successfully delivered are marked as 'sent', and those
            that fail to be deliver are marked as 'exception', and the
            corresponding error mail is output in the server logs.

            :param bool auto_commit: whether to force a commit of the mail status
                after sending each mail (meant only for scheduler processing);
                should never be True during normal transactions (default: False)
            :param bool raise_exception: whether to raise an exception if the
                email sending process has failed
            :return: True
        """
        for server_id, batch_ids in self._split_by_server():
            smtp_session = None
            try:
                smtp_session = self.env['ir.mail_server'].connect(mail_server_id=server_id)
                self.browse(batch_ids)._send(
                    auto_commit=auto_commit,
                    raise_exception=raise_exception,
                    smtp_session=smtp_session)
                _logger.info(
                    'Sent batch %s emails via mail server ID #%s',
                    len(batch_ids), server_id)
            finally:
                if smtp_session:
                    smtp_session.quit()

    @api.multi
    def _send(self, auto_commit=False, raise_exception=False, smtp_session=None):
        IrMailServer = self.env['ir.mail_server']
        for mail_id in self.ids:
            try:
                mail = self.browse(mail_id)
                # TDE note: remove me when model_id field is present on mail.message - done here to avoid doing it multiple times in the sub method
                if mail.model:
                    model = self.env['ir.model']._get(mail.model)[0]
                else:
                    model = None
                if model:
                    mail = mail.with_context(model_name=model.name)

                # load attachment binary data with a separate read(), as prefetching all
                # `datas` (binary field) could bloat the browse cache, triggerring
                # soft/hard mem limits with temporary data.
                attachments = [(a['datas_fname'], base64.b64decode(a['datas']))
                               for a in mail.attachment_ids.sudo().read(['datas_fname', 'datas'])]

                # specific behavior to customize the send email for notified partners
                email_list = []
                if mail.email_to:
                    email_list.append(mail.send_get_email_dict())
                for partner in mail.recipient_ids:
                    email_list.append(mail.send_get_email_dict(partner=partner))

                # headers
                headers = {}
                ICP = self.env['ir.config_parameter'].sudo()
                bounce_alias = ICP.get_param("mail.bounce.alias")
                catchall_domain = ICP.get_param("mail.catchall.domain")
                if bounce_alias and catchall_domain:
                    if mail.model and mail.res_id:
                        headers['Return-Path'] = '%s+%d-%s-%d@%s' % (bounce_alias, mail.id, mail.model, mail.res_id, catchall_domain)
                    else:
                        headers['Return-Path'] = '%s+%d@%s' % (bounce_alias, mail.id, catchall_domain)
                if mail.headers:
                    try:
                        headers.update(safe_eval(mail.headers))
                    except Exception:
                        pass

                # Writing on the mail object may fail (e.g. lock on user) which
                # would trigger a rollback *after* actually sending the email.
                # To avoid sending twice the same email, provoke the failure earlier
                mail.write({
                    'state': 'exception',
                    'failure_reason': _('Error without exception. Probably due do sending an email without computed recipients.'),
                })
                mail_sent = False

                # build an RFC2822 email.message.Message object and send it without queuing
                res = None
                for email in email_list:
                    msg = IrMailServer.build_email(
                        email_from=mail.email_from,
                        email_to=email.get('email_to'),
                        subject=mail.subject,
                        body=email.get('body'),
                        body_alternative=email.get('body_alternative'),
                        email_cc=tools.email_split(mail.email_cc),
                        reply_to=mail.reply_to,
                        attachments=attachments,
                        message_id=mail.message_id,
                        references=mail.references,
                        object_id=mail.res_id and ('%s-%s' % (mail.res_id, mail.model)),
                        subtype='html',
                        subtype_alternative='plain',
                        headers=headers)
                    try:
                        res = IrMailServer.send_email(
                            msg, mail_server_id=mail.mail_server_id.id, smtp_session=smtp_session)
                    except AssertionError as error:
                        if error.message == IrMailServer.NO_VALID_RECIPIENT:
                            # No valid recipient found for this particular
                            # mail item -> ignore error to avoid blocking
                            # delivery to next recipients, if any. If this is
                            # the only recipient, the mail will show as failed.
                            _logger.info("Ignoring invalid recipients for mail.mail %s: %s",
                                         mail.message_id, email.get('email_to'))
                        else:
                            raise
                if res:
                    mail.write({'state': 'sent', 'message_id': res, 'failure_reason': False})
                    mail_sent = True

                # /!\ can't use mail.state here, as mail.refresh() will cause an error
                # see revid:odo@openerp.com-20120622152536-42b2s28lvdv3odyr in 6.1
                if mail_sent:
                    _logger.info('Mail with ID %r and Message-Id %r successfully sent', mail.id, mail.message_id)
                mail._postprocess_sent_message(mail_sent=mail_sent)
            except MemoryError:
                # prevent catching transient MemoryErrors, bubble up to notify user or abort cron job
                # instead of marking the mail as failed
                _logger.exception(
                    'MemoryError while processing mail with ID %r and Msg-Id %r. Consider raising the --limit-memory-hard startup option',
                    mail.id, mail.message_id)
                raise
            except psycopg2.Error:
                # If an error with the database occurs, chances are that the cursor is unusable.
                # This will lead to an `psycopg2.InternalError` being raised when trying to write
                # `state`, shadowing the original exception and forbid a retry on concurrent
                # update. Let's bubble it.
                raise
            except Exception as e:
                failure_reason = tools.ustr(e)
                _logger.exception('failed sending mail (id: %s) due to %s', mail.id, failure_reason)
                mail.write({'state': 'exception', 'failure_reason': failure_reason})
                mail._postprocess_sent_message(mail_sent=False)
                if raise_exception:
                    if isinstance(e, AssertionError):
                        # get the args of the original error, wrap into a value and throw a MailDeliveryException
                        # that is an except_orm, with name and value as arguments
                        value = '. '.join(e.args)
                        raise MailDeliveryException(_("Mail Delivery Failed"), value)
                    raise

            if auto_commit is True:
                self._cr.commit()
        return True
<commit_msg>[FIX] mail: improve exception management when sending emails<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import datetime
import logging
import psycopg2
import threading

from collections import defaultdict
from email.utils import formataddr

from odoo import _, api, fields, models
from odoo import tools
from odoo.addons.base.ir.ir_mail_server import MailDeliveryException
from odoo.tools.safe_eval import safe_eval

_logger = logging.getLogger(__name__)

class MailMail(models.Model):
    """ Model holding RFC2822 email messages to send. This model also provides
        facilities to queue and send new email messages.  """
    _name = 'mail.mail'
    _description = 'Outgoing Mails'
    _inherits = {'mail.message': 'mail_message_id'}
    _order = 'id desc'
    _rec_name = 'subject'

    # content
    mail_message_id = fields.Many2one('mail.message', 'Message', required=True, ondelete='cascade', index=True, auto_join=True)
    body_html = fields.Text('Rich-text Contents', help="Rich-text/HTML message")
    references = fields.Text('References', help='Message references, such as identifiers of previous messages', readonly=1)
    headers = fields.Text('Headers', copy=False)
    # Auto-detected based on create() - if 'mail_message_id' was passed then this mail is a notification
    # and during unlink() we will not cascade delete the parent and its attachments
    notification = fields.Boolean('Is Notification', help='Mail has been created to notify people of an existing mail.message')
    # recipients
    email_to = fields.Text('To', help='Message recipients (emails)')
    email_cc = fields.Char('Cc', help='Carbon copy message recipients')
    recipient_ids = fields.Many2many('res.partner', string='To (Partners)')
    # process
    state = fields.Selection([
        ('outgoing', 'Outgoing'),
        ('sent', 'Sent'),
        ('received', 'Received'),
        ('exception', 'Delivery Failed'),
        ('cancel', 'Cancelled'),
    ], 'Status', readonly=True, copy=False, default='outgoing')
    auto_delete = fields.Boolean(
        'Auto Delete',
        help="Permanently delete this email after sending it, to save space")
    failure_reason = fields.Text(
        'Failure Reason', readonly=1,
        help="Failure reason. This is usually the exception thrown by the email server, stored to ease the debugging of mailing issues.")
    scheduled_date = fields.Char('Scheduled Send Date',
        help="If set, the queue manager will send the email after the date. If not set, the email will be send as soon as possible.")

    @api.model
    def create(self, values):
        # notification field: if not set, set if mail comes from an existing mail.message
        if 'notification' not in values and values.get('mail_message_id'):
            values['notification'] = True
        if not values.get('mail_message_id'):
            self = self.with_context(message_create_from_mail_mail=True)
        return super(MailMail, self).create(values)

    @api.multi
    def unlink(self):
        # cascade-delete the parent message for all mails that are not created for a notification
        to_cascade = self.search([('notification', '=', False), ('id', 'in', self.ids)]).mapped('mail_message_id')
        res = super(MailMail, self).unlink()
        to_cascade.unlink()
        return res

    @api.model
    def default_get(self, fields):
        # protection for `default_type` values leaking from menu action context (e.g. for invoices)
        # To remove when automatic context propagation is removed in web client
        if self._context.get('default_type') not in type(self).message_type.base_field.selection:
            self = self.with_context(dict(self._context, default_type=None))
        return super(MailMail, self).default_get(fields)

    @api.multi
    def mark_outgoing(self):
        return self.write({'state': 'outgoing'})

    @api.multi
    def cancel(self):
        return self.write({'state': 'cancel'})

    @api.model
    def process_email_queue(self, ids=None):
        """Send immediately queued messages, committing after each
           message is sent - this is not transactional and should
           not be called during another transaction!

           :param list ids: optional list of emails ids to send. If passed
                            no search is performed, and these ids are used
                            instead.
           :param dict context: if a 'filters' key is present in context,
                                this value will be used as an additional
                                filter to further restrict the outgoing
                                messages to send (by default all 'outgoing'
                                messages are sent).
        """
        if not self.ids:
            filters = ['&',
                       ('state', '=', 'outgoing'),
                       '|',
                       ('scheduled_date', '<', datetime.datetime.now()),
                       ('scheduled_date', '=', False)]
            if 'filters' in self._context:
                filters.extend(self._context['filters'])
            # TODO: make limit configurable
            ids = self.search(filters, limit=10000).ids
        res = None
        try:
            # auto-commit except in testing mode
            auto_commit = not getattr(threading.currentThread(), 'testing', False)
            res = self.browse(ids).send(auto_commit=auto_commit)
        except Exception:
            _logger.exception("Failed processing mail queue")
        return res

    @api.multi
    def _postprocess_sent_message(self, mail_sent=True):
        """Perform any post-processing necessary after sending ``mail``
        successfully, including deleting it completely along with its
        attachment if the ``auto_delete`` flag of the mail was set.
        Overridden by subclasses for extra post-processing behaviors.

        :param browse_record mail: the mail that was just sent
        :return: True
        """
        notif_emails = self.filtered(lambda email: email.notification)
        if notif_emails:
            notifications = self.env['mail.notification'].search([
                ('mail_message_id', 'in', notif_emails.mapped('mail_message_id').ids),
                ('is_email', '=', True)])
            if mail_sent:
                notifications.write({
                    'email_status': 'sent',
                })
            else:
                notifications.write({
                    'email_status': 'exception',
                })
        if mail_sent:
            self.sudo().filtered(lambda self: self.auto_delete).unlink()
        return True

    # ------------------------------------------------------
    # mail_mail formatting, tools and send mechanism
    # ------------------------------------------------------

    @api.multi
    def send_get_mail_body(self, partner=None):
        """Return a specific ir_email body. The main purpose of this method
        is to be inherited to add custom content depending on some module."""
        self.ensure_one()
        body = self.body_html or ''
        return body

    @api.multi
    def send_get_mail_to(self, partner=None):
        """Forge the email_to with the following heuristic:
          - if 'partner', recipient specific (Partner Name <email>)
          - else fallback on mail.email_to splitting """
        self.ensure_one()
        if partner:
            email_to = [formataddr((partner.name, partner.email))]
        else:
            email_to = tools.email_split_and_format(self.email_to)
        return email_to

    @api.multi
    def send_get_email_dict(self, partner=None):
        """Return a dictionary for specific email values, depending on a
        partner, or generic to the whole recipients given by mail.email_to.

            :param browse_record mail: mail.mail browse_record
            :param browse_record partner: specific recipient partner
        """
        self.ensure_one()
        body = self.send_get_mail_body(partner=partner)
        body_alternative = tools.html2plaintext(body)
        res = {
            'body': body,
            'body_alternative': body_alternative,
            'email_to': self.send_get_mail_to(partner=partner),
        }
        return res

    @api.multi
    def _split_by_server(self):
        """Returns an iterator of pairs `(mail_server_id, record_ids)` for current recordset.

        The same `mail_server_id` may repeat in order to limit batch size according to
        the `mail.session.batch.size` system parameter.
        """
        groups = defaultdict(list)
        # Turn prefetch OFF to avoid MemoryError on very large mail queues, we only care
        # about the mail server ids in this case.
        for mail in self.with_context(prefetch_fields=False):
            groups[mail.mail_server_id.id].append(mail.id)
        sys_params = self.env['ir.config_parameter'].sudo()
        batch_size = int(sys_params.get_param('mail.session.batch.size', 1000))
        for server_id, record_ids in groups.iteritems():
            for mail_batch in tools.split_every(batch_size, record_ids):
                yield server_id, mail_batch

    @api.multi
    def send(self, auto_commit=False, raise_exception=False):
        """ Sends the selected emails immediately, ignoring their current
            state (mails that have already been sent should not be passed
            unless they should actually be re-sent).
            Emails successfully delivered are marked as 'sent', and those
            that fail to be deliver are marked as 'exception', and the
            corresponding error mail is output in the server logs.

            :param bool auto_commit: whether to force a commit of the mail status
                after sending each mail (meant only for scheduler processing);
                should never be True during normal transactions (default: False)
            :param bool raise_exception: whether to raise an exception if the
                email sending process has failed
            :return: True
        """
        for server_id, batch_ids in self._split_by_server():
            smtp_session = None
            try:
                smtp_session = self.env['ir.mail_server'].connect(mail_server_id=server_id)
            except Exception as exc:
                if raise_exception:
                    # To be consistent and backward compatible with mail_mail.send() raised
                    # exceptions, it is encapsulated into an Odoo MailDeliveryException
                    raise MailDeliveryException(_('Unable to connect to SMTP Server'), exc)
                else:
                    self.browse(batch_ids).write({'state': 'exception', 'failure_reason': exc})
            else:
                self.browse(batch_ids)._send(
                    auto_commit=auto_commit,
                    raise_exception=raise_exception,
                    smtp_session=smtp_session)
                _logger.info(
                    'Sent batch %s emails via mail server ID #%s',
                    len(batch_ids), server_id)
            finally:
                if smtp_session:
                    smtp_session.quit()

    @api.multi
    def _send(self, auto_commit=False, raise_exception=False, smtp_session=None):
        IrMailServer = self.env['ir.mail_server']
        for mail_id in self.ids:
            try:
                mail = self.browse(mail_id)
                # TDE note: remove me when model_id field is present on mail.message - done here to avoid doing it multiple times in the sub method
                if mail.model:
                    model = self.env['ir.model']._get(mail.model)[0]
                else:
                    model = None
                if model:
                    mail = mail.with_context(model_name=model.name)

                # load attachment binary data with a separate read(), as prefetching all
                # `datas` (binary field) could bloat the browse cache, triggerring
                # soft/hard mem limits with temporary data.
                attachments = [(a['datas_fname'], base64.b64decode(a['datas']))
                               for a in mail.attachment_ids.sudo().read(['datas_fname', 'datas'])]

                # specific behavior to customize the send email for notified partners
                email_list = []
                if mail.email_to:
                    email_list.append(mail.send_get_email_dict())
                for partner in mail.recipient_ids:
                    email_list.append(mail.send_get_email_dict(partner=partner))

                # headers
                headers = {}
                ICP = self.env['ir.config_parameter'].sudo()
                bounce_alias = ICP.get_param("mail.bounce.alias")
                catchall_domain = ICP.get_param("mail.catchall.domain")
                if bounce_alias and catchall_domain:
                    if mail.model and mail.res_id:
                        headers['Return-Path'] = '%s+%d-%s-%d@%s' % (bounce_alias, mail.id, mail.model, mail.res_id, catchall_domain)
                    else:
                        headers['Return-Path'] = '%s+%d@%s' % (bounce_alias, mail.id, catchall_domain)
                if mail.headers:
                    try:
                        headers.update(safe_eval(mail.headers))
                    except Exception:
                        pass

                # Writing on the mail object may fail (e.g. lock on user) which
                # would trigger a rollback *after* actually sending the email.
                # To avoid sending twice the same email, provoke the failure earlier
                mail.write({
                    'state': 'exception',
                    'failure_reason': _('Error without exception. Probably due do sending an email without computed recipients.'),
                })
                mail_sent = False

                # build an RFC2822 email.message.Message object and send it without queuing
                res = None
                for email in email_list:
                    msg = IrMailServer.build_email(
                        email_from=mail.email_from,
                        email_to=email.get('email_to'),
                        subject=mail.subject,
                        body=email.get('body'),
                        body_alternative=email.get('body_alternative'),
                        email_cc=tools.email_split(mail.email_cc),
                        reply_to=mail.reply_to,
                        attachments=attachments,
                        message_id=mail.message_id,
                        references=mail.references,
                        object_id=mail.res_id and ('%s-%s' % (mail.res_id, mail.model)),
                        subtype='html',
                        subtype_alternative='plain',
                        headers=headers)
                    try:
                        res = IrMailServer.send_email(
                            msg, mail_server_id=mail.mail_server_id.id, smtp_session=smtp_session)
                    except AssertionError as error:
                        if error.message == IrMailServer.NO_VALID_RECIPIENT:
                            # No valid recipient found for this particular
                            # mail item -> ignore error to avoid blocking
                            # delivery to next recipients, if any. If this is
                            # the only recipient, the mail will show as failed.
                            _logger.info("Ignoring invalid recipients for mail.mail %s: %s",
                                         mail.message_id, email.get('email_to'))
                        else:
                            raise
                if res:
                    mail.write({'state': 'sent', 'message_id': res, 'failure_reason': False})
                    mail_sent = True

                # /!\ can't use mail.state here, as mail.refresh() will cause an error
                # see revid:odo@openerp.com-20120622152536-42b2s28lvdv3odyr in 6.1
                if mail_sent:
                    _logger.info('Mail with ID %r and Message-Id %r successfully sent', mail.id, mail.message_id)
                mail._postprocess_sent_message(mail_sent=mail_sent)
            except MemoryError:
                # prevent catching transient MemoryErrors, bubble up to notify user or abort cron job
                # instead of marking the mail as failed
                _logger.exception(
                    'MemoryError while processing mail with ID %r and Msg-Id %r. Consider raising the --limit-memory-hard startup option',
                    mail.id, mail.message_id)
                raise
            except psycopg2.Error:
                # If an error with the database occurs, chances are that the cursor is unusable.
                # This will lead to an `psycopg2.InternalError` being raised when trying to write
                # `state`, shadowing the original exception and forbid a retry on concurrent
                # update. Let's bubble it.
                raise
            except Exception as e:
                failure_reason = tools.ustr(e)
                _logger.exception('failed sending mail (id: %s) due to %s', mail.id, failure_reason)
                mail.write({'state': 'exception', 'failure_reason': failure_reason})
                mail._postprocess_sent_message(mail_sent=False)
                if raise_exception:
                    if isinstance(e, AssertionError):
                        # get the args of the original error, wrap into a value and throw a MailDeliveryException
                        # that is an except_orm, with name and value as arguments
                        value = '. '.join(e.args)
                        raise MailDeliveryException(_("Mail Delivery Failed"), value)
                    raise

            if auto_commit is True:
                self._cr.commit()
        return True
<|endoftext|>


<commit_before>from __future__ import with_statement

import logging
try:
    import simplejson as json
except ImportError:
    import json

logger = logging.getLogger('dispatcher.webservice.TaskController')

from tornado.web import HTTPError
from octopus.core.framework import ResourceNotFoundError, BaseResource, queue
from octopus.core.enums.node import *
from octopus.dispatcher.model import TaskGroup

ALLOWED_STATUS_VALUES = (NODE_DONE, NODE_CANCELED)

class BadStatusValueResponse(HTTPError):

    error_message = u'Status value not allowed (must be one of %s)' % (u", ".join([unicode(val) for val in ALLOWED_STATUS_VALUES]))

    def __init__(self):
        super(BadStatusValueResponse, self).__init__(400, self.error_message)

class TaskNotFoundError(ResourceNotFoundError):
    '''Raised when a requested task does not exist'''

class TasksResource(BaseResource):
    @queue
    def get(self):
        tasks = self.getDispatchTree().tasks
        tasks = [task.to_json() for task in tasks.values()]
        data = {'tasks': tasks}
        body = json.dumps(data)
        self.writeCallback(body)

class DeleteTasksResource(BaseResource):
    @queue
    def post(self):
        data = self.getBodyAsJSON()
        try:
            taskids = data['taskids']
        except:
            return HTTPError(400, 'Missing entry: "taskids".')
        else:
            taskidsList = taskids.split(",")
            for taskId in taskidsList:
                taskId = int(taskId)
                try:
                    task = self.getDispatchTree().tasks[taskId]
                except KeyError:
                    logger.warning("a pb occured with task %s" % str(taskId))
                    raise TaskNotFoundError(taskId)
                if task.nodes.values()[0].status not in ALLOWED_STATUS_VALUES:
                    logger.warning("bad status for task %s" % str(taskId))
                    return BadStatusValueResponse()
                task.archive()
            self.writeCallback("Tasks archived successfully.")

class TaskResource(BaseResource):
    @queue
    def get(self, taskID):
        taskID = int(taskID)
        task = self._findTask(taskID)
        odict = {'tasks': [task.to_json()]}
        body = json.dumps(odict)
        self.writeCallback(body)

    def _findTask(self, taskId):
        taskId = int(taskId)
        try:
            return self.getDispatchTree().tasks[taskId]
        except KeyError:
            raise TaskNotFoundError(taskId)

class TaskCommentResource(TaskResource):
    @queue
    def put(self, taskId):
        '''
        Sets a comment on the task.
        '''
        data = self.getBodyAsJSON()
        try:
            comment = data['comment']
        except:
            return HTTPError(400, 'Missing entry: "comment".')
        else:
            taskId = int(taskId)
            task = self._findTask(taskId)
            task.tags["comment"] = comment
            self.dispatcher.dispatchTree.toModifyElements.append(task)

class TaskUserResource(TaskResource):
    @queue
    def put(self, taskId):
        '''
        Sets the user of a task.
        '''
        data = self.getBodyAsJSON()
        try:
            user = data['user']
        except:
            return HTTPError(400, 'Missing entry: "user".')
        else:
            taskId = int(taskId)
            task = self._findTask(taskId)
            task.user = str(user)
            self.dispatcher.dispatchTree.toModifyElements.append(task)

class TaskEnvResource(TaskResource):
    @queue
    def post(self, taskId):
        taskId = int(taskId)
        task = self._findTask(taskId)
        data = self.getBodyAsJSON()
        env = data['environment']
        task.environment.clear()
        task.environment.update(env)
        message = "Environment of task %d has successfully been set." % taskId
        self.writeCallback(message)

    @queue
    def put(self, taskId):
        taskId = int(taskId)
        data = self.getBodyAsJSON()
        if not 'environment' in data:
            return HTTPError(400, "Missing 'environment' key in data.")
        task = self._findTask(taskId)
        env = data['environment']
        task.environment.clear()
        task.environment.update(env)
        message = "Environment of task %d has successfully been updated." % taskId
        self.writeCallback(message)

class TaskArgumentResource(TaskResource):
    @queue
    def post(self, taskId):
        taskId = int(taskId)
        data = self.getBodyAsJSON()
        if not 'environment' in data:
            return HTTPError(400, "Missing 'environment' key in data.")
        try:
            task = self._findTask(taskId)
        except TaskNotFoundError:
            return HTTPError(404, "Task not found. No such task %d" % taskId)
        arguments = data['arguments']
        task.arguments.clear()
        task.arguments.update(arguments)
        message = "Arguments of task %d have successfully been set." % taskId
        self.writeCallback(message)

    @queue
    def put(self, taskId):
        taskId = int(taskId)
        task = self._findTask(taskId)
        data = self.getBodyAsJSON()
        arguments = data['arguments']
        task.arguments.update(arguments)
        message = "Arguments of task %d have successfully been updated." % taskId
        self.writeCallback(message)

class TaskCommandResource(TaskResource):
    @queue
    def get(self, taskId):
        taskId = int(taskId)
        query = self.getBodyAsJSON()
        if 'status' in query:
            filterfunc = lambda command: command.status in [int(s) for s in query['status']]
        else:
            filterfunc = lambda command: True
        try:
            body, rootTaskId = self.filteredTask(taskId, filterfunc)
            body = json.dumps({'commands': body, 'rootTask': rootTaskId})
        except TaskNotFoundError:
            return HTTPError(404, "No such task. Task %d was not found." % taskId)
        else:
            self.writeCallback(body)

    def filteredTask(self, taskId, filterfunc):
        root = self._findTask(taskId)
        commands = []
        tasks = [root]
        while tasks:
            task = tasks.pop(0)
            if isinstance(task, TaskGroup):
                for child in task.tasks:
                    tasks.append(child)
            else:
                commands += [c for c in task.commands if filterfunc(c)]
        while root.parent:
            root = root.parent
        return [c.to_json() for c in commands], root.id

class TaskTreeResource(TaskResource):
    @queue
    def get(self, taskId):
        taskId = int(taskId)
        try:
            data = self.getSubTasks(taskId)
        except RuntimeError, e:
            if e.args[0][0] is TaskNotFoundError:
                return HTTPError(404, "Task %d not found." % taskId)
        else:
            self.writeCallback(data)

    def getSubTasks(self, rootTaskId):
        rootTask = self._findTask(rootTaskId)
        tasks = []
        commands = []
        unprocessed = [rootTask]
        while unprocessed:
            task = unprocessed.pop()
            tasks.append(task.to_json())
            if isinstance(task, TaskGroup):
                unprocessed += task.tasks
            else:
                commands += [command.to_json() for command in task.commands]
        return {
            'tasks': tasks,
            'commands': commands
        }
<commit_msg>Don't block on keyerror while archiving a list of tasks<commit_after>from __future__ import with_statement

import logging
try:
    import simplejson as json
except ImportError:
    import json

logger = logging.getLogger('dispatcher.webservice.TaskController')

from tornado.web import HTTPError
from octopus.core.framework import ResourceNotFoundError, BaseResource, queue
from octopus.core.enums.node import *
from octopus.dispatcher.model import TaskGroup

ALLOWED_STATUS_VALUES = (NODE_DONE, NODE_CANCELED)

class BadStatusValueResponse(HTTPError):

    error_message = u'Status value not allowed (must be one of %s)' % (u", ".join([unicode(val) for val in ALLOWED_STATUS_VALUES]))

    def __init__(self):
        super(BadStatusValueResponse, self).__init__(400, self.error_message)

class TaskNotFoundError(ResourceNotFoundError):
    '''Raised when a requested task does not exist'''

class TasksResource(BaseResource):
    @queue
    def get(self):
        tasks = self.getDispatchTree().tasks
        tasks = [task.to_json() for task in tasks.values()]
        data = {'tasks': tasks}
        body = json.dumps(data)
        self.writeCallback(body)

class DeleteTasksResource(BaseResource):
    @queue
    def post(self):
        data = self.getBodyAsJSON()
        try:
            taskids = data['taskids']
        except:
            return HTTPError(400, 'Missing entry: "taskids".')
        else:
            taskidsList = taskids.split(",")
            for taskId in taskidsList:
                taskId = int(taskId)
                try:
                    task = self.getDispatchTree().tasks[taskId]
                except KeyError:
                    logger.warning("Trying to archive task %s that no longer exists." % str(taskId))
                    continue
                if task.nodes.values()[0].status not in ALLOWED_STATUS_VALUES:
                    logger.warning("bad status for task %s" % str(taskId))
                    return BadStatusValueResponse()
                task.archive()
            self.writeCallback("Tasks archived successfully.")

class TaskResource(BaseResource):
    @queue
    def get(self, taskID):
        taskID = int(taskID)
        task = self._findTask(taskID)
        odict = {'tasks': [task.to_json()]}
        body = json.dumps(odict)
        self.writeCallback(body)

    def _findTask(self, taskId):
        taskId = int(taskId)
        try:
            return self.getDispatchTree().tasks[taskId]
        except KeyError:
            raise TaskNotFoundError(taskId)

class TaskCommentResource(TaskResource):
    @queue
    def put(self, taskId):
        '''
        Sets a comment on the task.
        '''
        data = self.getBodyAsJSON()
        try:
            comment = data['comment']
        except:
            return HTTPError(400, 'Missing entry: "comment".')
        else:
            taskId = int(taskId)
            task = self._findTask(taskId)
            task.tags["comment"] = comment
            self.dispatcher.dispatchTree.toModifyElements.append(task)

class TaskUserResource(TaskResource):
    @queue
    def put(self, taskId):
        '''
        Sets the user of a task.
        '''
        data = self.getBodyAsJSON()
        try:
            user = data['user']
        except:
            return HTTPError(400, 'Missing entry: "user".')
        else:
            taskId = int(taskId)
            task = self._findTask(taskId)
            task.user = str(user)
            self.dispatcher.dispatchTree.toModifyElements.append(task)

class TaskEnvResource(TaskResource):
    @queue
    def post(self, taskId):
        taskId = int(taskId)
        task = self._findTask(taskId)
        data = self.getBodyAsJSON()
        env = data['environment']
        task.environment.clear()
        task.environment.update(env)
        message = "Environment of task %d has successfully been set." % taskId
        self.writeCallback(message)

    @queue
    def put(self, taskId):
        taskId = int(taskId)
        data = self.getBodyAsJSON()
        if not 'environment' in data:
            return HTTPError(400, "Missing 'environment' key in data.")
        task = self._findTask(taskId)
        env = data['environment']
        task.environment.clear()
        task.environment.update(env)
        message = "Environment of task %d has successfully been updated." % taskId
        self.writeCallback(message)

class TaskArgumentResource(TaskResource):
    @queue
    def post(self, taskId):
        taskId = int(taskId)
        data = self.getBodyAsJSON()
        if not 'environment' in data:
            return HTTPError(400, "Missing 'environment' key in data.")
        try:
            task = self._findTask(taskId)
        except TaskNotFoundError:
            return HTTPError(404, "Task not found. No such task %d" % taskId)
        arguments = data['arguments']
        task.arguments.clear()
        task.arguments.update(arguments)
        message = "Arguments of task %d have successfully been set." % taskId
        self.writeCallback(message)

    @queue
    def put(self, taskId):
        taskId = int(taskId)
        task = self._findTask(taskId)
        data = self.getBodyAsJSON()
        arguments = data['arguments']
        task.arguments.update(arguments)
        message = "Arguments of task %d have successfully been updated." % taskId
        self.writeCallback(message)

class TaskCommandResource(TaskResource):
    @queue
    def get(self, taskId):
        taskId = int(taskId)
        query = self.getBodyAsJSON()
        if 'status' in query:
            filterfunc = lambda command: command.status in [int(s) for s in query['status']]
        else:
            filterfunc = lambda command: True
        try:
            body, rootTaskId = self.filteredTask(taskId, filterfunc)
            body = json.dumps({'commands': body, 'rootTask': rootTaskId})
        except TaskNotFoundError:
            return HTTPError(404, "No such task. Task %d was not found." % taskId)
        else:
            self.writeCallback(body)

    def filteredTask(self, taskId, filterfunc):
        root = self._findTask(taskId)
        commands = []
        tasks = [root]
        while tasks:
            task = tasks.pop(0)
            if isinstance(task, TaskGroup):
                for child in task.tasks:
                    tasks.append(child)
            else:
                commands += [c for c in task.commands if filterfunc(c)]
        while root.parent:
            root = root.parent
        return [c.to_json() for c in commands], root.id

class TaskTreeResource(TaskResource):
    @queue
    def get(self, taskId):
        taskId = int(taskId)
        try:
            data = self.getSubTasks(taskId)
        except RuntimeError, e:
            if e.args[0][0] is TaskNotFoundError:
                return HTTPError(404, "Task %d not found." % taskId)
        else:
            self.writeCallback(data)

    def getSubTasks(self, rootTaskId):
        rootTask = self._findTask(rootTaskId)
        tasks = []
        commands = []
        unprocessed = [rootTask]
        while unprocessed:
            task = unprocessed.pop()
            tasks.append(task.to_json())
            if isinstance(task, TaskGroup):
                unprocessed += task.tasks
            else:
                commands += [command.to_json() for command in task.commands]
        return {
            'tasks': tasks,
            'commands': commands
        }
<|endoftext|>


<commit_before># coding=utf-8
try:
    import boto3
    boto3_available = True
except ImportError:
    boto3_available = False

import os

from .alerter import Alerter

class SESAlerter(Alerter):
    """Send email alerts using Amazon's SES service."""

    def __init__(self, config_options):
        if not boto3_available:
            print("Boto3 package is not available, cannot use SESAlerter.")
            return

        Alerter.__init__(self, config_options)
        try:
            from_addr = config_options["from"]
            to_addr = config_options["to"]
        except Exception:
            raise RuntimeError("Required configuration fields missing")

        if from_addr == "":
            raise RuntimeError("missing mail from address")
        if to_addr == "":
            raise RuntimeError("missing mail to address")

        self.from_addr = from_addr
        self.to_addr = to_addr

        self.support_catchup = True

        self.ses_client_params = {}

        if 'aws_region' in config_options:
            os.environ["AWS_DEFAULT_REGION"] = config_options['aws_region']

        if 'aws_access_key' in config_options and 'aws_secret_key' in config_options:
            self.ses_client_params['aws_access_key_id'] = config_options['aws_access_key']
            self.ses_client_params['aws_secret_access_key'] = config_options['aws_secret_key']

    def send_alert(self, name, monitor):
        """Send the email."""

        type = self.should_alert(monitor)
        (days, hours, minutes, seconds) = self.get_downtime(monitor)

        if monitor.is_remote():
            host = " on %s " % monitor.running_on
        else:
            host = " on host %s" % self.hostname

        mail = {'Source': self.from_addr}
        mail['Destination'] = {'ToAddresses': [self.to_addr]}

        if type == "":
            return
        elif type == "failure":
            message = {'Subject': {'Data': "[%s] Monitor %s Failed!" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': """Monitor %s%s has failed.
            Failed at: %s
            Downtime: %d+%02d:%02d:%02d
            Virtual failure count: %d
            Additional info: %s
            Description: %s""" % (
                name,
                host,
                self.format_datetime(monitor.first_failure_time()),
                days, hours, minutes, seconds,
                monitor.virtual_fail_count(),
                monitor.get_result(),
                monitor.describe())
            }}
            try:
                if monitor.recover_info != "":
                    message['Body']['Text']['Data'] += "\nRecovery info: %s" % monitor.recover_info
            except AttributeError:
                message['Body']['Text']['Data'] += "\nNo recovery info available"

        elif type == "success":
            message = {'Subject': {'Data': "[%s] Monitor %s succeeded" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, self.format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe())}}

        elif type == "catchup":
            message = {'Subject': {'Data': "[%s] Monitor %s failed earlier!" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, self.format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe())}}

        else:
            print("Unknown alert type %s" % type)
            return

        mail['Message'] = message

        if not self.dry_run:
            try:
                client = boto3.client('ses', **self.ses_client_params)
                client.send_email(**mail)
            except Exception as e:
                print("Couldn't send mail: %s" % e)
                self.available = False
        else:
            print("dry_run: would send email:")
            print("Subject: %s" % message['Subject']['Data'])
            print("Body: %s" % message['Body']['Text']['Data'])
<commit_msg>Update ses alerter to get_config_option<commit_after># coding=utf-8
try:
    import boto3
    boto3_available = True
except ImportError:
    boto3_available = False

import os

from .alerter import Alerter

class SESAlerter(Alerter):
    """Send email alerts using Amazon's SES service."""

    def __init__(self, config_options):
        if not boto3_available:
            print("Boto3 package is not available, cannot use SESAlerter.")
            return

        Alerter.__init__(self, config_options)
        self.from_addr = Alerter.get_config_option(
            config_options,
            'from',
            allow_empty=False
        )
        self.to_addr = Alerter.get_config_option(
            config_options,
            'to',
            allow_empty=False
        )

        self.support_catchup = True

        self.ses_client_params = {}

        aws_region = Alerter.get_config_option(config_options, 'aws_region')
        if aws_region:
            os.environ["AWS_DEFAULT_REGION"] = aws_region

        aws_access_key = Alerter.get_config_option(config_options, 'aws_access_key')
        aws_secret_key = Alerter.get_config_option(config_options, 'aws_secret_access_key')

        if aws_access_key and aws_secret_key:
            self.ses_client_params['aws_access_key_id'] = aws_access_key
            self.ses_client_params['aws_secret_access_key'] = aws_secret_key

    def send_alert(self, name, monitor):
        """Send the email."""

        type = self.should_alert(monitor)
        (days, hours, minutes, seconds) = self.get_downtime(monitor)

        if monitor.is_remote():
            host = " on %s " % monitor.running_on
        else:
            host = " on host %s" % self.hostname

        mail = {'Source': self.from_addr}
        mail['Destination'] = {'ToAddresses': [self.to_addr]}

        if type == "":
            return
        elif type == "failure":
            message = {'Subject': {'Data': "[%s] Monitor %s Failed!" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': """Monitor %s%s has failed.
            Failed at: %s
            Downtime: %d+%02d:%02d:%02d
            Virtual failure count: %d
            Additional info: %s
            Description: %s""" % (
                name,
                host,
                self.format_datetime(monitor.first_failure_time()),
                days, hours, minutes, seconds,
                monitor.virtual_fail_count(),
                monitor.get_result(),
                monitor.describe())
            }}
            try:
                if monitor.recover_info != "":
                    message['Body']['Text']['Data'] += "\nRecovery info: %s" % monitor.recover_info
            except AttributeError:
                message['Body']['Text']['Data'] += "\nNo recovery info available"

        elif type == "success":
            message = {'Subject': {'Data': "[%s] Monitor %s succeeded" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, self.format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe())}}

        elif type == "catchup":
            message = {'Subject': {'Data': "[%s] Monitor %s failed earlier!" % (self.hostname, name)}}
            message['Body'] = {'Text': {'Data': "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, self.format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe())}}

        else:
            print("Unknown alert type %s" % type)
            return

        mail['Message'] = message

        if not self.dry_run:
            try:
                client = boto3.client('ses', **self.ses_client_params)
                client.send_email(**mail)
            except Exception as e:
                print("Couldn't send mail: %s" % e)
                self.available = False
        else:
            print("dry_run: would send email:")
            print("Subject: %s" % message['Subject']['Data'])
            print("Body: %s" % message['Body']['Text']['Data'])
<|endoftext|>


<commit_before><commit_msg>3以上のフォルダに対応<commit_after><|endoftext|>


<commit_before>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		UseExceptions()
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))
		os.system("cp %s %s" % (ehdr, "~/projects/dg9/downsamp.hdr") )

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA

		# ensure decent data is retrieved
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0) # ignore max as NODATA is 0
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<commit_msg>Removed temp code leftover from earlier testing.<commit_after>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		UseExceptions()
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA

		# ensure decent data is retrieved
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0) # ignore max as NODATA is 0
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<|endoftext|>