<commit_before># -*- coding: utf-8 -*-
import codecs
import os
from collections import defaultdict

from copy import deepcopy
from pyparsing import Literal, White, Word, alphanums, CharsNotIn
from pyparsing import Forward, Group, Optional, OneOrMore, ZeroOrMore
from pyparsing import pythonStyleComment, Empty, Combine
from setuptools.glob import iglob

class MySQLParser(object):

    key = Word(alphanums + "_-")
    space = White().suppress()
    value = CharsNotIn("\n")
    filename = Literal("!includedir") + Word(alphanums + " /.")
    comment = ("#")
    config_entry = (key + Optional(space)
                    + Optional(
                        Literal("=").suppress() + Optional(space)
                        + Optional(value) + Optional(space)
                        + Optional("#")
                    )
                    )
    single_value = key
    client_block = Forward()
    client_block << Group((Literal("[").suppress()
                          + key
                          + Literal("]").suppress())
                          + Group(ZeroOrMore(Group(config_entry)))
                          )

    include_block = Forward()
    include_block << Group(Combine(filename) +
                           Group(Group(Empty())))

    # The file consists of client_blocks and include_files
    client_file = OneOrMore( include_block| client_block ).ignore(
        pythonStyleComment)

    file_header = """# File parsed and saved by privacyidea.\n\n"""

    def __init__(self, infile="/etc/mysql/my.cnf",
                 content=None):
        self.file = None
        if content:
            self.content = content
        else:
            self.file = infile
            self._read()

    def _read(self):
        """
        Reread the contents from the disk
        """
        f = codecs.open(self.file, "r", "utf-8")
        self.content = f.read()
        f.close()

    def get(self):
        """
        return the grouped config
        """
        if self.file:
            self._read()
        config = self.client_file.parseString(self.content)
        return config

    def format(self, dict_config):
        '''
        :return: The formatted data as it would be written to a file
        '''
        output = ""
        output += self.file_header
        for section, attributes in dict_config.items():
            if section.startswith("!includedir"):
                output += "{0}\n".format(section)
            else:
                output += "[{0}]\n".format(section)
                for k, v in attributes.iteritems():
                    if v:
                        output += "{k} = {v}\n".format(k=k, v=v)
                    else:
                        output += "{k}\n".format(k=k)

            output += "\n"

        return output

    def get_dict(self, section=None, key=None):
        '''
        return the client config as a dictionary.
        '''
        ret = {}
        config = self.get()
        for client in config:
            client_config = {}
            for attribute in client[1]:
                if len(attribute) > 1:
                    client_config[attribute[0]] = attribute[1]
                elif len(attribute) == 1:
                    client_config[attribute[0]] = None
            ret[client[0]] = client_config
        if section:
            ret = ret.get(section, {})
            if key:
                ret = ret.get(key)
        return ret

    def save(self, dict_config=None, outfile=None):
        if dict_config:
            output = self.format(dict_config)
            f = codecs.open(outfile, 'w', 'utf-8')
            for line in output.splitlines():
                f.write(line + "\n")
            f.close()

class MySQLConfiguration(object):
    """
    an object that takes care of collecting the mysql configuration from all included directories

    assumption: a value can be uniquely mapped to a single file! (i.e. the options
    are not overwritten in multiple files)
    """
    def __init__(self, root_filename, parser_cls=MySQLParser):
        """
        :param root_filename: filename of the configuration file
        :param parser_cls: you may pass another class than `MySQLParser` here, e.g. to edit files remotely.
        """
        self.parser_cls = parser_cls
        self.root = parser_cls(root_filename)
        #: map a key (given as (section, key)) to a MySQLConfiguration instance.
        #: if a key has no entry in _key_map, it is managed in `root`
        self._key_map = {}
        self._children = []
        self._read_config()

    def _read_child_config(self, filename):
        """
        Read configuration from *filename*, store MySQLConfiguration instances in self._children
        and self._key_map
        """
        child_config = MySQLConfiguration(filename, self.parser_cls)
        self._children.append(child_config)
        for section, contents in child_config.get_dict().iteritems():
            for key, value in contents.iteritems():
                location = (section, key)
                if location in self._key_map:
                    raise RuntimeError('Value {!r}/{!r} already found in {!r}'.format(section, value,
                                                                                self._key_map[location].root.file))
                self._key_map[location] = child_config

    def _read_config(self):
        """
        This initializes `_key_map` and _children.
        """
        self._key_map = {}
        self._children = []
        root_dct = self.root.get_dict()
        base_directory = os.path.dirname(self.root.file)
        for section, contents in root_dct.iteritems():
            # find all !includedir lines, add configuration to self._children and self._sectionmap
            if section.startswith('!includedir'):
                relative_directory = section.split(' ', 1)[1]
                directory = os.path.abspath(os.path.join(base_directory, relative_directory))
                # include all files in the directory
                for filename in iglob(os.path.join(directory, '*.cnf')):
                    # order is not guaranteed, according to mysql docs
                    # parse every file, return parsing result
                    self._read_child_config(filename)
            elif section.startswith('!'):
                raise NotImplementedError()

    def get_dict(self, section=None, key=None):
        """
        Get the MySQL configuration, with !includedir entries purged, and with all included files processed.
        :return:
        """
        dct = self.root.get_dict()
        for dct_key in dct.keys():
            if dct_key.startswith('!'):
                del dct[dct_key]
        # add children
        for child in self._children:
            dct.update(child.get_dict())
        ret = dct
        if section:
            ret = dct.get(section, {})
            if key:
                ret = ret.get(key)
        return ret

    def save(self, new_config):
        """
        write all the config objects in *config* to the right files
        """
        config = deepcopy(new_config)
        # iterate over all values of which we know that they belong in another file
        #: maps MySQLConfiguration objects to configuration
        save_configs = defaultdict(lambda: defaultdict(dict))
        # find all keys which are updated in `config` and delegated to another file
        for (child_section, child_key), child in self._key_map.iteritems():
            # check if `config` contains a value for `child_section` and `child_key`
            if child_section in config:
                child_contents = config[child_section]
                if child_key in child_contents:
                    # delegate the value to the MySQLConfiguration object
                    save_configs[child][child_section][child_key] = config[child_section][child_key]
                    # We have handled this value, remove it from `config`
                    del config[child_section][child_key]
                    # Delete empty sections from `config`
                    if not config[child_section]:
                        del config[child_section]
        # find all keys that are *added* to sections for which entries in _value_map exist
        # (i.e. the section is delegated to another config file)
        # if the section has values in multiple files, one is chosen non-deterministically
        for (child_section, child_key), child in self._key_map.iteritems():
            if child_section in config:
                save_configs[child][child_section].update(config[child_section])
                del config[child_section]
        # all remaining keys in `config` belong in `root`.
        # now, save all children
        for child, child_config in save_configs.iteritems():
            child.save(child_config)
        # add all !includedir directives
        root_dct = self.root.get_dict()
        for key, value in root_dct.iteritems():
            if key.startswith('!'):
                config[key] = value
        # save the local configuration
        self.root.save(config, self.root.file)
        # re-read configuration!
        self._read_config()<commit_msg>Allow a customized opener function<commit_after># -*- coding: utf-8 -*-
import codecs
import os
from collections import defaultdict

from copy import deepcopy
from pyparsing import Literal, White, Word, alphanums, CharsNotIn
from pyparsing import Forward, Group, Optional, OneOrMore, ZeroOrMore
from pyparsing import pythonStyleComment, Empty, Combine
from setuptools.glob import iglob

class MySQLParser(object):

    key = Word(alphanums + "_-")
    space = White().suppress()
    value = CharsNotIn("\n")
    filename = Literal("!includedir") + Word(alphanums + " /.")
    comment = ("#")
    config_entry = (key + Optional(space)
                    + Optional(
                        Literal("=").suppress() + Optional(space)
                        + Optional(value) + Optional(space)
                        + Optional("#")
                    )
                    )
    single_value = key
    client_block = Forward()
    client_block << Group((Literal("[").suppress()
                          + key
                          + Literal("]").suppress())
                          + Group(ZeroOrMore(Group(config_entry)))
                          )

    include_block = Forward()
    include_block << Group(Combine(filename) +
                           Group(Group(Empty())))

    # The file consists of client_blocks and include_files
    client_file = OneOrMore( include_block| client_block ).ignore(
        pythonStyleComment)

    file_header = """# File parsed and saved by privacyidea.\n\n"""

    def __init__(self, infile="/etc/mysql/my.cnf",
                 content=None,
                 opener=open):
        self.file = None
        self.opener = opener
        if content:
            self.content = content
        else:
            self.file = infile
            self._read()

    def _read(self):
        """
        Reread the contents from the disk
        """
        with self.opener(self.file, 'rb') as f:
            self.content = f.read().decode('utf-8')

    def get(self):
        """
        return the grouped config
        """
        if self.file:
            self._read()
        config = self.client_file.parseString(self.content)
        return config

    def format(self, dict_config):
        '''
        :return: The formatted data as it would be written to a file
        '''
        output = ""
        output += self.file_header
        for section, attributes in dict_config.items():
            if section.startswith("!includedir"):
                output += "{0}\n".format(section)
            else:
                output += "[{0}]\n".format(section)
                for k, v in attributes.iteritems():
                    if v:
                        output += "{k} = {v}\n".format(k=k, v=v)
                    else:
                        output += "{k}\n".format(k=k)

            output += "\n"

        return output

    def get_dict(self, section=None, key=None):
        '''
        return the client config as a dictionary.
        '''
        ret = {}
        config = self.get()
        for client in config:
            client_config = {}
            for attribute in client[1]:
                if len(attribute) > 1:
                    client_config[attribute[0]] = attribute[1]
                elif len(attribute) == 1:
                    client_config[attribute[0]] = None
            ret[client[0]] = client_config
        if section:
            ret = ret.get(section, {})
            if key:
                ret = ret.get(key)
        return ret

    def save(self, dict_config=None, outfile=None):
        if dict_config:
            output = self.format(dict_config)
            with self.opener(outfile, 'wb') as f:
                for line in output.splitlines():
                    f.write(line.encode('utf-8') + "\n")

class MySQLConfiguration(object):
    """
    an object that takes care of collecting the mysql configuration from all included directories

    assumption: a value can be uniquely mapped to a single file! (i.e. the options
    are not overwritten in multiple files)
    """
    def __init__(self, root_filename, opener=open):
        """
        :param root_filename: filename of the configuration file
        :param open: you may pass another function to open files here, e.g. to edit files remotely.
        """
        self._opener = opener
        self.root = MySQLParser(root_filename, opener=opener)
        #: map a key (given as (section, key)) to a MySQLConfiguration instance.
        #: if a key has no entry in _key_map, it is managed in `root`
        self._key_map = {}
        self._children = []
        self._read_config()

    def _read_child_config(self, filename):
        """
        Read configuration from *filename*, store MySQLConfiguration instances in self._children
        and self._key_map
        """
        child_config = MySQLConfiguration(filename, self._opener)
        self._children.append(child_config)
        for section, contents in child_config.get_dict().iteritems():
            for key, value in contents.iteritems():
                location = (section, key)
                if location in self._key_map:
                    raise RuntimeError('Value {!r}/{!r} already found in {!r}'.format(section, value,
                                                                                self._key_map[location].root.file))
                self._key_map[location] = child_config

    def _read_config(self):
        """
        This initializes `_key_map` and _children.
        """
        self._key_map = {}
        self._children = []
        root_dct = self.root.get_dict()
        base_directory = os.path.dirname(self.root.file)
        for section, contents in root_dct.iteritems():
            # find all !includedir lines, add configuration to self._children and self._sectionmap
            if section.startswith('!includedir'):
                relative_directory = section.split(' ', 1)[1]
                directory = os.path.abspath(os.path.join(base_directory, relative_directory))
                # include all files in the directory
                for filename in iglob(os.path.join(directory, '*.cnf')):
                    # order is not guaranteed, according to mysql docs
                    # parse every file, return parsing result
                    self._read_child_config(filename)
            elif section.startswith('!'):
                raise NotImplementedError()

    def get_dict(self, section=None, key=None):
        """
        Get the MySQL configuration, with !includedir entries purged, and with all included files processed.
        :return:
        """
        dct = self.root.get_dict()
        for dct_key in dct.keys():
            if dct_key.startswith('!'):
                del dct[dct_key]
        # add children
        for child in self._children:
            dct.update(child.get_dict())
        ret = dct
        if section:
            ret = dct.get(section, {})
            if key:
                ret = ret.get(key)
        return ret

    def save(self, new_config):
        """
        write all the config objects in *config* to the right files
        """
        config = deepcopy(new_config)
        # iterate over all values of which we know that they belong in another file
        #: maps MySQLConfiguration objects to configuration
        save_configs = defaultdict(lambda: defaultdict(dict))
        # find all keys which are updated in `config` and delegated to another file
        for (child_section, child_key), child in self._key_map.iteritems():
            # check if `config` contains a value for `child_section` and `child_key`
            if child_section in config:
                child_contents = config[child_section]
                if child_key in child_contents:
                    # delegate the value to the MySQLConfiguration object
                    save_configs[child][child_section][child_key] = config[child_section][child_key]
                    # We have handled this value, remove it from `config`
                    del config[child_section][child_key]
                    # Delete empty sections from `config`
                    if not config[child_section]:
                        del config[child_section]
        # find all keys that are *added* to sections for which entries in _value_map exist
        # (i.e. the section is delegated to another config file)
        # if the section has values in multiple files, one is chosen non-deterministically
        for (child_section, child_key), child in self._key_map.iteritems():
            if child_section in config:
                save_configs[child][child_section].update(config[child_section])
                del config[child_section]
        # all remaining keys in `config` belong in `root`.
        # now, save all children
        for child, child_config in save_configs.iteritems():
            child.save(child_config)
        # add all !includedir directives
        root_dct = self.root.get_dict()
        for key, value in root_dct.iteritems():
            if key.startswith('!'):
                config[key] = value
        # save the local configuration
        self.root.save(config, self.root.file)
        # re-read configuration!
        self._read_config()<|endoftext|>


<commit_before>
import re
import time
from subprocess import Popen, PIPE, STDOUT

from stratuslab.pdiskbackend.utils import abort, print_detail

#######################################################
# Class representing a command passed to the back-end #
#######################################################

class CommandRunner(object):
    cmd_output_start = '<<<<<<<<<<'
    cmd_output_end = '>>>>>>>>>>'

    RETRY_ERRORS = [(255, re.compile('^Connection to .* closed by remote host.', re.MULTILINE)),
                    (1, re.compile('^ssh: connect to host .* Connection refused', re.MULTILINE)),
                    (255, re.compile('^ssh: connect to host .* No route to host', re.MULTILINE))]
    MAX_RETRIES = 3

    def __init__(self, action, cmd, successMsgs=[], failureOkMsgs=[]):
        self.action = action
        self.action_cmd = cmd
        self.successMsgs = successMsgs or []
        self.failureOkMsgs = failureOkMsgs or []
        self.proc = None

    def execute(self):
        status = 0
        self.debug("Executing command: '%s'" % (' '.join(self.action_cmd)), 1)
        try:
            self.proc = Popen(self.action_cmd, shell=False, stdout=PIPE, stderr=STDOUT)
        except OSError, details:
            abort('Failed to execute %s action: %s' % (self.action, details))
            status = 1
        return status

    def checkStatus(self):
        optInfo = ()
        try:
            retcode, output = self._getStatusOutputOrRetry(self.action)
            output = self._filter_command_output(output)
            if retcode != 0 and len(output) != 0:
                self.debug("ERROR: %s action, exit code %s. Command output:\n%s\n%s\n%s" % \
                           (self.action, retcode, self.cmd_output_start, output, self.cmd_output_end))
                # In some cases we are OK when failure happens.
                for failurePattern in self.failureOkMsgs:
                    output_regexp = re.compile(failurePattern, re.MULTILINE)
                    matcher = output_regexp.search(output)
                    if matcher:
                        retcode = 0
                        self.debug('... But we are OK to proceed. Setting retcode to 0.')
                        break
            else:
                # Need to check if the command is expected to return an output when successful
                success = False
                if self.successMsgs:
                    for successPattern in self.successMsgs:
                        output_regexp = re.compile(successPattern, re.MULTILINE)
                        matcher = output_regexp.search(output)
                        if matcher:
                            # Return only the first capturing group
                            if output_regexp.groups > 0:
                                optInfo = matcher.groups()
                            success = True
                            break
                else:
                    if len(output) == 0:
                        success = True
                if success:
                    self.debug("SUCCESS: %s action completed successfully." % self.action, 1)
                    if len(output) > 0:
                        self.debug('Command output:\n%s\n%s\n%s' % (self.cmd_output_start, output, self.cmd_output_end), 2)
                else:
                    self.debug("ERROR: %s action, exit code %s. But a failure case detected after parsing the output. Command output:\n%s\n%s\n%s" % \
                              (self.action, retcode, self.cmd_output_start, output, self.cmd_output_end))
                    retcode = -1
                    self.debug('exit code was reset to %i' % retcode)
        except OSError as ex:
            abort('Failed to execute %s action: %s' % (self.action, ex))

        if self.action in ['map', 'delete'] and retcode == 255 and not output.strip():
            retcode = 0
            self.debug('map and delete actions (command exited with 255 and no output returned) - exit code was reset to %i.' % retcode)

        if retcode == 255:
            if self.action in ['map', 'delete'] and not output.strip():
                retcode = 0
                self.debug('map and delete actions (no output returned) - exit code was reset to %i.' % retcode)
            if self.action in ['unmap']:
                retcode = 0
                self.debug('unmap action - exit code was reset to %i.' % retcode)

        return retcode, optInfo

    def _filter_command_output(self, output):
        lines = []
        for line in output.split('\n'):
            line = line.strip().strip('\r').strip()
            if not line:
                continue
            if line.startswith('Warning: Permanently added'):
                continue
            # NetApp Clustered may return a bell... crazy.
            if line.startswith('\x07'):
                continue
            lines.append(line)
        return '\n'.join(lines)

    def _getStatusOutputOrRetry(self, _action=''):
        """_action parameter is required only for testability: for mocking the method
        with 'side_effect'."""
        retcode, output = self._getStatusOutput()
        return self._retryOnError(retcode, output)

    def _getStatusOutput(self):
        retcode = self.proc.wait()
        return retcode, self.proc.communicate()[0]

    def _retryOnError(self, retcode, output):
        retries = 0
        while self._needToRetry(retcode, output) and retries < self.MAX_RETRIES:
            time.sleep(1)
            self.execute()
            retcode, output = self._getStatusOutput()
            retries += 1
        return retcode, output

    def _needToRetry(self, retcode, output):
        if retcode == 0:
            return False
        for rc, re_out in self.RETRY_ERRORS:
            if rc == retcode and re_out.match(output):
                return True
        return False

    def debug(self, msg, level=0):
        print_detail(msg, level)
<commit_msg>updated the logic around defining success of the command that was just ran.<commit_after>
import re
import time
from subprocess import Popen, PIPE, STDOUT

from stratuslab.pdiskbackend.utils import abort, print_detail

#######################################################
# Class representing a command passed to the back-end #
#######################################################

class CommandRunner(object):
    cmd_output_start = '<<<<<<<<<<'
    cmd_output_end = '>>>>>>>>>>'

    RETRY_ERRORS = [(255, re.compile('^Connection to .* closed by remote host.', re.MULTILINE)),
                    (1, re.compile('^ssh: connect to host .* Connection refused', re.MULTILINE)),
                    (255, re.compile('^ssh: connect to host .* No route to host', re.MULTILINE))]
    MAX_RETRIES = 3

    def __init__(self, action, cmd, successMsgs=[], failureOkMsgs=[]):
        self.action = action
        self.action_cmd = cmd
        self.successMsgs = successMsgs or []
        self.failureOkMsgs = failureOkMsgs or []
        self.proc = None

    def execute(self):
        status = 0
        self.debug("Executing command: '%s'" % (' '.join(self.action_cmd)), 1)
        try:
            self.proc = Popen(self.action_cmd, shell=False, stdout=PIPE, stderr=STDOUT)
        except OSError, details:
            abort('Failed to execute %s action: %s' % (self.action, details))
            status = 1
        return status

    def checkStatus(self):
        optInfo = ()
        try:
            retcode, output = self._getStatusOutputOrRetry(self.action)
            output = self._filter_command_output(output)
            if retcode != 0 and len(output) != 0:
                self.debug("ERROR: %s action, exit code %s. Command output:\n%s\n%s\n%s" % \
                           (self.action, retcode, self.cmd_output_start, output, self.cmd_output_end))
                # In some cases we are OK when failure happens.
                for failurePattern in self.failureOkMsgs:
                    output_regexp = re.compile(failurePattern, re.MULTILINE)
                    matcher = output_regexp.search(output)
                    if matcher:
                        retcode = 0
                        self.debug('... But we are OK to proceed. Setting retcode to 0.')
                        break
            else:
                # Need to check if the command is expected to return an output when successful
                success = False
                if len(output) == 0:
                    success = True
                if self.successMsgs and len(output) > 0:
                    success = False
                    for successPattern in self.successMsgs:
                        output_regexp = re.compile(successPattern, re.MULTILINE)
                        matcher = output_regexp.search(output)
                        if matcher:
                            # Return only the first capturing group
                            if output_regexp.groups > 0:
                                optInfo = matcher.groups()
                            success = True
                            break
                if success:
                    self.debug("SUCCESS: %s action completed successfully." % self.action, 1)
                    if len(output) > 0:
                        self.debug('Command output:\n%s\n%s\n%s' % (self.cmd_output_start, output, self.cmd_output_end), 2)
                else:
                    self.debug("ERROR: %s action, exit code %s. But a failure case detected after parsing the output. Command output:\n%s\n%s\n%s" % \
                              (self.action, retcode, self.cmd_output_start, output, self.cmd_output_end))
                    retcode = -1
                    self.debug('exit code was reset to %i' % retcode)
        except OSError as ex:
            abort('Failed to execute %s action: %s' % (self.action, ex))

        if self.action in ['map', 'delete'] and retcode == 255 and not output.strip():
            retcode = 0
            self.debug('map and delete actions (command exited with 255 and no output returned) - exit code was reset to %i.' % retcode)

        if retcode == 255:
            if self.action in ['map', 'delete'] and not output.strip():
                retcode = 0
                self.debug('map and delete actions (no output returned) - exit code was reset to %i.' % retcode)
            if self.action in ['unmap']:
                retcode = 0
                self.debug('unmap action - exit code was reset to %i.' % retcode)

        return retcode, optInfo

    def _filter_command_output(self, output):
        lines = []
        for line in output.split('\n'):
            line = line.strip().strip('\r').strip()
            if not line:
                continue
            if line.startswith('Warning: Permanently added'):
                continue
            # NetApp Clustered may return a bell... crazy.
            if line.startswith('\x07'):
                continue
            lines.append(line)
        return '\n'.join(lines)

    def _getStatusOutputOrRetry(self, _action=''):
        """_action parameter is required only for testability: for mocking the method
        with 'side_effect'."""
        retcode, output = self._getStatusOutput()
        return self._retryOnError(retcode, output)

    def _getStatusOutput(self):
        retcode = self.proc.wait()
        return retcode, self.proc.communicate()[0]

    def _retryOnError(self, retcode, output):
        retries = 0
        while self._needToRetry(retcode, output) and retries < self.MAX_RETRIES:
            time.sleep(1)
            self.execute()
            retcode, output = self._getStatusOutput()
            retries += 1
        return retcode, output

    def _needToRetry(self, retcode, output):
        if retcode == 0:
            return False
        for rc, re_out in self.RETRY_ERRORS:
            if rc == retcode and re_out.match(output):
                return True
        return False

    def debug(self, msg, level=0):
        print_detail(msg, level)
<|endoftext|>


<commit_before>__author__ = 'kuhn'

import networkx
import codecs
import logging
import os
from lxml import etree
import sys
import mimetypes

# TODO: define paths in csv files
DGDROOT = "dgd2_data"
RESOURCEPROXIES = "ResourceProxyList"
SPEAKERXPATH = "//InEvent/Event"
RESOURCEPATH = "dgd2_data/dgd2cmdi/cmdiOutput/"
PREFIX = 'cmdi_'
NAME = 'AGD'

# this is the landing page prefix for the agd werbservice
LANDINGPG = 'http://dgd.ids-mannheim.de/service/DGD2Web/ExternalAccessServlet?command=displayData&id='

class ResourceTreeCollection(networkx.MultiDiGraph):
    """
    represent resources in a tree structure. all involved paths
    The resurce collection is a Digraph representation of all data.
    each data is a node that holds a dict defining their attributes:

    repopath : <path locates the metadata file in the dgd repository scopus>
    corpusroot : <boolean. is the file a corpus catalogue or not?>

    input data paths must be pointed towards already cmdi transformed data.
    """

    def __init__(self, corpuspath, eventspath, speakerspath):
        super(ResourceTreeCollection, self).__init__()
        corpusnames = os.listdir(corpuspath)
        eventcorpusnames = os.listdir(eventspath)
        speakercorpusnames = os.listdir(speakerspath)
        self.name = NAME
        # cwdstart = os.getcwd()

        # define a collection root that precedes all corpora
        self.add_node("AGD_root")

        if cmp(corpusnames, eventcorpusnames) and cmp(eventcorpusnames,
                                                      speakercorpusnames):
            logging.info("Resources are aligned")
            print corpusnames
            print eventcorpusnames
            print speakercorpusnames

        else:
            logging.error("Resources are not aligned. Check paths and/ or"
                          "consistency of resource subsets")

        # define corpus nodes

        for corpus in corpusnames:
            # iterate over all corpus file names in corpus metadata dir
            # and define a node for each.
            try:
                etr = etree.parse(corpuspath+'/'+corpus)

            except (etree.XMLSyntaxError, IOError):
                logging.error("Warning. xml file was not parsed: "+corpus)
                continue
            self.add_node(
                corpus.split('_')[1].rstrip('-'),
                {
                    'repopath': self.contextpath(corpus, DGDROOT),
                    'corpusroot': True,
                    'type': 'metadata',
                    'etreeobject': etr,
                    'filename': corpus}

            )
            # add edge from root to current node
            self.add_edge('AGD_root', corpus.split('_')[1].rstrip('-'))

        # define event nodes and add add them to their corpus root

        for event in eventcorpusnames:

            """
            define eventnodes. note that each event has a number of sessions that
            are not explicitly stored as nodes.
            """

            if self.has_node(event):
                eventcorpusfilepath = eventspath + '/' + event

                for filename in os.listdir(eventcorpusfilepath):
                    try:
                        etr = etree.parse(eventcorpusfilepath + '/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue

                    eventnodename = filename.split('.')[0].lstrip(PREFIX).rstrip('_extern')
                    self.add_node(eventnodename, {
                        'repopath': self.contextpath(event, DGDROOT),
                        'corpusroot': False,
                        'type': 'metadata',
                        'etreeobject': etr,
                        'filename': filename}
                    )
                    self.add_edge(event, eventnodename)
            # finally connect an event to all speakers that take part in it.
            for speaker in self.find_speakers(event):
                self.add_edge(event, speaker)

        # define speaker nodes and add them to their corpus root

        for speakercorp in speakercorpusnames:

            """
            define speakernodes. note that each speaker is part of an session
            in an event. to just find the event, a speaker has taken part,
            access the <InEvent> element of the speaker cmdi metadata.
            """

            # find corpus the speaker is part of
            if self.has_node(speakercorp):
                speakercorpusfilepath = speakerspath + '/' + speakercorp

                for filename in os.listdir(speakercorpusfilepath):
                    try:
                        etr = etree.parse(speakercorpusfilepath+'/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue
                    speakernodename = filename.split('.')[0].lstrip(PREFIX).rstrip('_extern')
                    self.add_node(speakernodename, {
                        'repopath': self.contextpath(speakercorp, DGDROOT),
                        'corpusroot': False,
                        'type': 'metadata',
                        'etreeobject': etr,
                        'filename': filename}
                    )
                    # define an edge from the parent corpus (speakercorp)
                    # to the current speakernode
                    # example: "PF" --> "PF--_S_00103.xml"
                    self.add_edge(speakercorp, speakernodename)
                    # define edges from events to current speaker

                    speakerevents = self.find_events(speakernodename)
                    for event in speakerevents:
                        self.add_edge(event, speakernodename)

    @staticmethod
    def contextpath(fname, startpath):
        # FIXME: method always returns None.
        """find the path from a startpath to a given file"""
        for root, dirs, files in os.walk(startpath):
            if fname in files:
                return os.path.join(root, fname)
            else:
                return None

    def find_eventsessions(self, speakernode):
        """
        :param speakernode
        :return list of event labels
        searches for events a speaker takes part in by looking up the
        metadata itself.
        Since there is no physical resource for sessions, they are not
        filed as nodes.
        """
        expression = "//InEvents/EventSession"
        inevent = self.node.get(speakernode).get("etreeobject").xpath(expression)
        # must add _extern label to find in graph
        # inevent_out = [event.text + '_extern.xml' for event in inevent]
        ineventsessions_out = [event.text for event in inevent]
        return ineventsessions_out

    def find_events(self, speakernode):
        """
        :param speakernode
        :return list of event labels
        searches for events a speaker takes part in by looking up the
        metadata itself.
        Since there is no physical resource for sessions, they are not
        filed as nodes.
        """
        sessions = self.find_eventsessions(speakernode)
        # take the session of event label and split it down to the event
        events = ['_'.join(str(segment) for segment in session.split('_')[0:3])
                  for session in sessions]
        events = list(set(events))
        return events

    def find_speakers(self, eventnode):
        """
        :param eventnode:
        :return: list of speaker labels fo/und in event
        """
        session = "//Session"
        speakerpath = "Speaker/Label"
        sessionsofevent = self.node.get(eventnode).get("etreeobject").xpath(session)
        speakerlabels = list()

        # add all sessions of the event as attribute
        self.node.get(eventnode).update({'sessions': sessionsofevent})

        for session in sessionsofevent:
            # must add "_extern" label to find speaker in graphh
            speakerlabels.extend([speaker.text for speaker in session.xpath(speakerpath)])
            # speakerlabels.extend([speaker.text + '_extern.xml' for speaker in session.xpath(speakerpath)])
        return speakerlabels

    def define_resourceproxy(self, metafilenode):
        """
        defines the ResourceProxies for all Resources referred via edges
               <Resources>
        <ResourceProxyList> </ResourceProxyList>
        <JournalFileProxyList> </JournalFileProxyList>
        <ResourceRelationList> </ResourceRelationList>
        </Resources>
        :param metafilenode:
        :return:
        """

        cmdi_etrobj = self.node.get(metafilenode).get("etreeobject")
        cmdiroot = cmdi_etrobj.getroot()
        resourceproxies = cmdiroot.find("Resources").find("ResourceProxyList")

        in_nodes = [i[0] for i in self.in_edges(metafilenode)]
        out_nodes = [i[1] for i in self.out_edges(metafilenode)]

        resource_nodes = out_nodes + in_nodes

        # remove the abstract node from the resource list
        if 'AGD_root' in resource_nodes:
            resource_nodes.remove('AGD_root')

        # make sure there are just unique references
        resource_nodes = set(resource_nodes)

        for node in resource_nodes:
            # where is the edge pointed to?
            # access the filename

            node_fname = self.node.get(node).get("filename")
            resourceproxy = etree.Element("ResourceProxy")
            resourceref = etree.SubElement(resourceproxy, "ResourceRef")
            resourcetype = etree.SubElement(resourceproxy, "ResourceType")
            resourceproxy.set("id", node)

            resourcetype.set("mimetype", mimetypes.guess_type(node_fname)[0])
            resourceref.text = LANDINGPG + node  # add absolute path

            resourceproxy = etree.SubElement(resourceproxy, "ResourceIsPart")
            resourceproxy.set("href", node_fname)

            # insert new resourceproxyelement in list
            resourceproxies.append(resourceproxy)

        # version resource info
        # isVersionOf = etree.SubElement(resourceproxies, "isVersionOf")

    def write_xml(self, nodename, fname):
        """
        write the xml of a node
        :param node:
        :return:
        """
        self.node.get(nodename).get('etreeobject').write(fname, encoding='utf-8', method='xml')

    def build_resourceproxy(self):
        """
        inplace resourceproxy generation of the resource.
        modifies the stored etree object of each resource.
        :param cmdifile:
        :return:
        """

        for resource in self.nodes_iter():

            # pass the node name to define_resourceproxy
            if resource is not None:
                self.define_resourceproxy(resource)

    def get_cmdi(self, nodename):
        """
        outputs the cmdi of the node as prettyprinted xml.
        :param outputpath:
        :return:
        """

        pass

    def build_geolocation(self, nodename):
        """
        inplace computation of the geolocations
        :return:
        """
        # TODO: find geolocation package to compute locations
        # from the provided grid coordinates

        # if provided, obtain grid coordinates.

        for node in self.nodes_iter():

            pass

<commit_msg>removed speaker refs from resourceproxies<commit_after>__author__ = 'kuhn'

import networkx
import codecs
import logging
import os
from lxml import etree
import sys
import mimetypes

# TODO: define paths in csv files
DGDROOT = "dgd2_data"
RESOURCEPROXIES = "ResourceProxyList"
SPEAKERXPATH = "//InEvent/Event"
RESOURCEPATH = "dgd2_data/dgd2cmdi/cmdiOutput/"
PREFIX = 'cmdi_'
NAME = 'AGD'

import urllib

# this is the landing page prefix for the agd werbservice
LANDINGPG = u"http://dgd.ids-mannheim.de/service/DGD2Web/ExternalAccessServlet?command=displayData&id="

class ResourceTreeCollection(networkx.MultiDiGraph):
    """
    represent resources in a tree structure. all involved paths
    The resurce collection is a Digraph representation of all data.
    each data is a node that holds a dict defining their attributes:

    repopath : <path locates the metadata file in the dgd repository scopus>
    corpusroot : <boolean. is the file a corpus catalogue or not?>

    input data paths must be pointed towards already cmdi transformed data.
    """

    def __init__(self, corpuspath, eventspath, speakerspath):
        super(ResourceTreeCollection, self).__init__()
        corpusnames = os.listdir(corpuspath)
        eventcorpusnames = os.listdir(eventspath)
        speakercorpusnames = os.listdir(speakerspath)
        self.name = NAME
        # cwdstart = os.getcwd()

        # define a collection root that precedes all corpora
        self.add_node("AGD_root")

        if cmp(corpusnames, eventcorpusnames) and cmp(eventcorpusnames,
                                                      speakercorpusnames):
            logging.info("Resources are aligned")
            print corpusnames
            print eventcorpusnames
            print speakercorpusnames

        else:
            logging.error("Resources are not aligned. Check paths and/ or"
                          "consistency of resource subsets")

        # define corpus nodes

        for corpus in corpusnames:
            # iterate over all corpus file names in corpus metadata dir
            # and define a node for each.
            try:
                etr = etree.parse(corpuspath+'/'+corpus)

            except (etree.XMLSyntaxError, IOError):
                logging.error("Warning. xml file was not parsed: "+corpus)
                continue
            self.add_node(
                corpus.split('_')[1].rstrip('-'),
                {
                    'repopath': self.contextpath(corpus, DGDROOT),
                    'corpusroot': True,
                    'type': 'metadata',
                    'etreeobject': etr,
                    'filename': corpus}

            )
            # add edge from root to current node
            self.add_edge('AGD_root', corpus.split('_')[1].rstrip('-'))

        # define event nodes and add add them to their corpus root

        for event in eventcorpusnames:

            """
            define eventnodes. note that each event has a number of sessions that
            are not explicitly stored as nodes.
            """

            if self.has_node(event):
                eventcorpusfilepath = eventspath + '/' + event

                for filename in os.listdir(eventcorpusfilepath):
                    try:
                        etr = etree.parse(eventcorpusfilepath + '/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue

                    eventnodename = filename.split('.')[0].lstrip(PREFIX).rstrip('_extern')
                    self.add_node(eventnodename, {
                        'repopath': self.contextpath(event, DGDROOT),
                        'corpusroot': False,
                        'type': 'metadata',
                        'etreeobject': etr,
                        'filename': filename}
                    )
                    self.add_edge(event, eventnodename)
            # finally connect an event to all speakers that take part in it.
            for speaker in self.find_speakers(event):
                self.add_edge(event, speaker)

        # define speaker nodes and add them to their corpus root

        for speakercorp in speakercorpusnames:

            """
            define speakernodes. note that each speaker is part of an session
            in an event. to just find the event, a speaker has taken part,
            access the <InEvent> element of the speaker cmdi metadata.
            """

            # find corpus the speaker is part of
            if self.has_node(speakercorp):
                speakercorpusfilepath = speakerspath + '/' + speakercorp

                for filename in os.listdir(speakercorpusfilepath):
                    try:
                        etr = etree.parse(speakercorpusfilepath+'/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue
                    speakernodename = filename.split('.')[0].lstrip(PREFIX).rstrip('_extern')
                    self.add_node(speakernodename, {
                        'repopath': self.contextpath(speakercorp, DGDROOT),
                        'corpusroot': False,
                        'type': 'metadata',
                        'etreeobject': etr,
                        'filename': filename}
                    )
                    # define an edge from the parent corpus (speakercorp)
                    # to the current speakernode
                    # example: "PF" --> "PF--_S_00103.xml"
                    self.add_edge(speakercorp, speakernodename)
                    # define edges from events to current speaker

                    speakerevents = self.find_events(speakernodename)
                    for event in speakerevents:
                        self.add_edge(event, speakernodename)

    @staticmethod
    def contextpath(fname, startpath):
        # FIXME: method always returns None.
        """find the path from a startpath to a given file"""
        for root, dirs, files in os.walk(startpath):
            if fname in files:
                return os.path.join(root, fname)
            else:
                return None

    def find_eventsessions(self, speakernode):
        """
        :param speakernode
        :return list of event labels
        searches for events a speaker takes part in by looking up the
        metadata itself.
        Since there is no physical resource for sessions, they are not
        filed as nodes.
        """
        expression = "//InEvents/EventSession"
        inevent = self.node.get(speakernode).get("etreeobject").xpath(expression)
        # must add _extern label to find in graph
        # inevent_out = [event.text + '_extern.xml' for event in inevent]
        ineventsessions_out = [event.text for event in inevent]
        return ineventsessions_out

    def find_events(self, speakernode):
        """
        :param speakernode
        :return list of event labels
        searches for events a speaker takes part in by looking up the
        metadata itself.
        Since there is no physical resource for sessions, they are not
        filed as nodes.
        """
        sessions = self.find_eventsessions(speakernode)
        # take the session of event label and split it down to the event
        events = ['_'.join(str(segment) for segment in session.split('_')[0:3])
                  for session in sessions]
        events = list(set(events))
        return events

    def find_speakers(self, eventnode):
        """
        :param eventnode:
        :return: list of speaker labels fo/und in event
        """
        session = "//Session"
        speakerpath = "Speaker/Label"
        sessionsofevent = self.node.get(eventnode).get("etreeobject").xpath(session)
        speakerlabels = list()

        # add all sessions of the event as attribute
        self.node.get(eventnode).update({'sessions': sessionsofevent})

        for session in sessionsofevent:
            # must add "_extern" label to find speaker in graphh
            speakerlabels.extend([speaker.text for speaker in session.xpath(speakerpath)])
            # speakerlabels.extend([speaker.text + '_extern.xml' for speaker in session.xpath(speakerpath)])
        return speakerlabels

    def define_resourceproxy(self, metafilenode):
        """
        defines the ResourceProxies for all Resources referred via edges
               <Resources>
        <ResourceProxyList> </ResourceProxyList>
        <JournalFileProxyList> </JournalFileProxyList>
        <ResourceRelationList> </ResourceRelationList>
        </Resources>
        :param metafilenode:
        :return:
        """

        cmdi_etrobj = self.node.get(metafilenode).get("etreeobject")
        cmdiroot = cmdi_etrobj.getroot()
        resourceproxies = cmdiroot.find("Resources").find("ResourceProxyList")

        in_nodes = [i[0] for i in self.in_edges(metafilenode)]
        out_nodes = [i[1] for i in self.out_edges(metafilenode)]

        resource_nodes = out_nodes + in_nodes

        # remove speaker references for now
        for speaker in self.find_speakers(metafilenode):
            if speaker in resource_nodes:
                resource_nodes.remove(speaker)

        # remove the abstract node from the resource list
        if 'AGD_root' in resource_nodes:
            resource_nodes.remove('AGD_root')
        if metafilenode in resource_nodes:
            resource_nodes.remove(metafilenode)
        # make sure there are just unique references
        resource_nodes = set(resource_nodes)

        for node in resource_nodes:
            # where is the edge pointed to?
            # access the filename

            node_fname = self.node.get(node).get("filename")
            resourceproxy = etree.SubElement(resourceproxies, "ResourceProxy")
            resourceref = etree.SubElement(resourceproxy, "ResourceRef")
            resourcetype = etree.SubElement(resourceproxy, "ResourceType")
            resourceproxy.set("id", node)

            resourcetype.set("mimetype", mimetypes.guess_type(node_fname)[0])
            landingpage = urllib.unquote(LANDINGPG)
            resourceref.text = node
            resourceref.set("href", landingpage + node)

            # Generate an is PartRelationship for backreference
            ispart = etree.SubElement(resourceproxy, "ResourceIsPart")

            # insert new resourceproxyelement in list
            # resourceproxies.append(resourceproxy)

        # version resource info
        # isVersionOf = etree.SubElement(resourceproxies, "isVersionOf")

    def write_xml(self, nodename, fname):
        """
        write the xml of a node
        :param node:
        :return:
        """
        self.node.get(nodename).get('etreeobject').write(fname, encoding='utf-8', method='xml')

    def build_resourceproxy(self):
        """
        inplace resourceproxy generation of the resource.
        modifies the stored etree object of each resource.
        :param cmdifile:
        :return:
        """

        for resource in self.nodes_iter():

            # pass the node name to define_resourceproxy
            if resource != 'AGD_root':
                self.define_resourceproxy(resource)

    def get_cmdi(self, nodename):
        """
        outputs the cmdi of the node as prettyprinted xml.
        :param outputpath:
        :return:
        """

        pass

    def build_geolocation(self, nodename):
        """
        inplace computation of the geolocations
        :return:
        """
        # TODO: find geolocation package to compute locations
        # from the provided grid coordinates

        # if provided, obtain grid coordinates.

        for node in self.nodes_iter():

            pass

<|endoftext|>


<commit_before>__author__ = 'kuhn'
<commit_msg>added DiGraphs for DGD Resources<commit_after>__author__ = 'kuhn'

import networkx
import codecs
import logging
import os
from lxml import etree

DGDROOT = "dgd2_data"

class ResourceTreeCollection(networkx.DiGraph):
    """
    represent resources in a tree structure. all involved paths
    The resurce collection is a Digraph representation of all data.
    each data is a node that holds a dict defining their attributes:

    repopath : <path locates the metadata file in the dgd repository scopus>
    corpusroot : <boolean. is the file a corpus catalogue or not?>

    input data paths must be pointed towards already cmdi transformed data.
    """

    def __init__(self, corpuspath, eventspath, speakerspath):
        super(ResourceTreeCollection, self).__init__()
        corpusNames = os.listdir(corpuspath)
        eventCorpusNames = os.listdir(eventspath)
        speakerCorpusNames = os.listdir(speakerspath)
        cwdStart = os.getcwd()
        if cmp(corpusNames, eventCorpusNames) and cmp(eventCorpusNames,
                                                     speakerCorpusNames):
            logging.info("Resources are aligned")
            print corpusNames
            print eventCorpusNames
            print speakerCorpusNames

        else:
            logging.error("Resources are not aligned. Check paths and/ or"
                          "consistency of resource subsets")

        # define corpus nodes

        for corpus in corpusNames:
            # iterate over all corpus file names in corpus metadata dir
            # and define a node for each.
            try:
                etr = etree.parse(corpuspath+'/'+corpus)
            except (etree.XMLSyntaxError, IOError):
                logging.error("Warning. xml file was not parsed: "+corpus)
                continue

            self.add_node(
                corpus.split('_')[0].rstrip('-'),
                {
                    'repopath': self.contextpath(corpus, DGDROOT),
                    'corpusroot': True,
                    'etreeobject': etr}

            )
        # define event nodes and add add them to their corpus root

        for event in eventCorpusNames:

            if self.has_node(event):
                eventcorpusfilepath = eventspath + '/' + event

                for filename in os.listdir(eventcorpusfilepath):
                    try:
                        etr = etree.parse(eventcorpusfilepath + '/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue

                    eventnodename = filename.split('.')[0]
                    self.add_node(eventnodename, {
                        'repopath': self.contextpath(event, DGDROOT),
                        'corpusroot': False,
                        'etreeobject': etr}
                    )
                    self.add_edge(event, eventnodename)

        # define speaker nodes and add them to their corpus root

        # change cwd to events directory

        for speaker in speakerCorpusNames:

            if self.has_node(speaker):
                speakercorpusfilepath = speakerspath + '/' + speaker

                for filename in os.listdir(speakercorpusfilepath):
                    try:
                        etr = etree.parse(speakercorpusfilepath+'/' + filename)
                    except (etree.XMLSyntaxError, IOError):
                        logging.error("Warning. xml file was not parsed: " +
                                      filename)
                        continue
                    speakernodename = filename.split('.')[0]
                    self.add_node(speakernodename, {
                        'repopath': self.contextpath(speaker, DGDROOT),
                        'corpusroot': False,
                        'etreeobject': etr}
                    )
                    self.add_edge(speaker, speakernodename)

        # add event ->speaker edges for speaker in events

    @staticmethod
    def contextpath(fname, startpath):
        # FIXME: method always returns None.
        """find the path from a startpath to a given file"""
        for root, dirs, files in os.walk(startpath):
            if fname in files:
                return os.path.join(root, fname)
            else:
                return None

    # TODO: find all speakers in an event

class ResourceTree(networkx.DiGraph):
    """
    a b-tree structure to link all local dgd resources, generated for
    each dgd metadata file.
    Starting point: ResourceBTree is a child of networkx.DiGraph
    """

    # TODO: Find a packages for tree representation or use networx

    def __init__(self, filename):
        super(ResourceTree).__init__()
        """
        init a resource tree object as directed graph:
        read in a cmdi-transformed dgd metafile
        :return:
        """
        self.etreeobject = None
        try:
            self.etreeobject = etree.parse(codecs.open(filename,
                                                       encoding="utf-8"))

        except IOError:
            logging.error("file not readable")

        fileroot = self.etreeobject.getroot()

        for element in fileroot.iter():
            print

<|endoftext|>


<commit_before>from datetime import datetime, date
from django.test import SimpleTestCase
from corehq.apps.reports_core.exceptions import FilterValueException
from corehq.apps.reports_core.filters import DatespanFilter, ChoiceListFilter, \
    NumericFilter, DynamicChoiceListFilter, Choice
from corehq.apps.userreports.exceptions import BadSpecError
from corehq.apps.userreports.reports.filters.values import SHOW_ALL_CHOICE, \
    CHOICE_DELIMITER, NumericFilterValue, DateFilterValue
from corehq.apps.userreports.reports.filters.factory import ReportFilterFactory
from corehq.apps.userreports.reports.filters.specs import ReportFilter
from dimagi.utils.dates import DateSpan

class FilterTestCase(SimpleTestCase):

    def test_no_type(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "field": "some_field",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_bad_type(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "invalid_type",
                "field": "some_field",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_missing_field(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "date",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_missing_slug(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "date",
                "field": "some_field",
                "display": "Some display name"
            })

    def test_translation(self):
        shared_conf = {
            "type": "date",
            "field": "some_field",
            "slug": "some_slug",
        }

        # Plain string
        conf = {"display": "foo"}
        conf.update(shared_conf)
        filter = ReportFilterFactory.from_spec(conf)
        self.assertEqual(filter.context(None, lang=None)['label'], "foo")
        self.assertEqual(filter.context(None, lang="fr")['label'], "foo")

        # Translation
        conf = {"display": {"en": "english", "fr": "french"}}
        conf.update(shared_conf)
        filter = ReportFilterFactory.from_spec(conf)
        self.assertEqual(filter.context(None, lang=None)['label'], "english")
        self.assertEqual(filter.context(None, lang="fr")['label'], "french")
        self.assertEqual(filter.context(None, lang="en")['label'], "english")
        self.assertEqual(filter.context(None, lang="es")['label'], "english")

class DateFilterTestCase(SimpleTestCase):

    def test_date_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "date",
            "field": "modified_on_field",
            "slug": "modified_on_slug",
            "display": "Date Modified"
        })
        self.assertEqual(DatespanFilter, type(filter))
        self.assertEqual('modified_on_slug', filter.name)
        self.assertEqual('Date Modified', filter.label)

    def test_compare_as_string_option(self):

        def get_query_value(compare_as_string):

            spec = {
                "type": "date",
                "field": "modified_on_field",
                "slug": "my_slug",
                "display": "date Modified",
                "compare_as_string": compare_as_string,
            }
            reports_core_filter = ReportFilterFactory.from_spec(spec)
            reports_core_value = reports_core_filter.get_value({
                "my_slug-start": "2015-06-07",
                "my_slug-end": "2015-06-08",
                "date_range_inclusive": True,
            })

            filter = ReportFilter.wrap(spec)
            return filter.create_filter_value(reports_core_value).to_sql_values()

        val = get_query_value(compare_as_string=False)
        self.assertEqual(type(val['my_slug_startdate']), datetime)
        self.assertEqual(type(val['my_slug_enddate']), datetime)

        val = get_query_value(compare_as_string=True)
        self.assertEqual(type(val['my_slug_startdate']), str)
        self.assertEqual(type(val['my_slug_enddate']), str)

class NumericFilterTestCase(SimpleTestCase):

    def test_numeric_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "numeric",
            "field": "number_of_children_field",
            "slug": "number_of_children_slug",
            "display": "Number of Children",
        })
        self.assertEqual(NumericFilter, type(filter))
        self.assertEqual("number_of_children_slug", filter.name)
        self.assertEqual("Number of Children", filter.label)

    def test_numeric_filter_value(self):
        filter = ReportFilter.wrap({
            "type": "numeric",
            "field": "number_of_children_field",
            "slug": "number_of_children_slug",
            "display": "Number of Children",
        })
        NumericFilterValue(filter, None)
        NumericFilterValue(filter, {'operator': '<', 'operand': 3})
        with self.assertRaises(AssertionError):
            NumericFilterValue(filter, {'operator': 'sql injection', 'operand': 3})

class ChoiceListFilterTestCase(SimpleTestCase):
    CHOICES = [
        {
            "value": "NEGATIVE",
            "display": "negative"
        },
        {
            "value": "POSITIVE",
            "display": "positive"
        }
    ]

    def test_choice_list_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "display": "Diagnosis",
            "choices": self.CHOICES,
            "show_all": False,
        })
        self.assertEqual(ChoiceListFilter, type(filter))
        self.assertEqual('diagnosis_slug', filter.name)
        self.assertEqual('Diagnosis', filter.label)
        self.assertEqual(2, len(filter.choices))
        for i, choice in enumerate(self.CHOICES):
            self.assertEqual(filter.choices[i].value, choice['value'])
            self.assertEqual(filter.choices[i].display, choice['display'])

        # check values
        self.assertEqual('positive', filter.value(diagnosis_slug='POSITIVE').display)

    def test_choice_list_filter_show_all(self):
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "display": "Diagnosis",
            "choices": self.CHOICES,
            "show_all": True,
        })
        self.assertEqual(3, len(filter.choices))

        self.assertEqual(SHOW_ALL_CHOICE, filter.choices[0].value)
        for i, choice in enumerate(self.CHOICES):
            self.assertEqual(filter.choices[i + 1].value, choice['value'])
            self.assertEqual(filter.choices[i + 1].display, choice['display'])

        # check all value
        self.assertEqual('Show all', filter.value(diagnosis_slug=SHOW_ALL_CHOICE).display)

    def test_choice_list_filter_with_integers(self):
        choices = [
            {
                "value": 0,
                "display": "negative"
            },
            {
                "value": 1,
                "display": "positive"
            }
        ]
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "datatype": "integer",
            "display": "Diagnosis",
            "choices": choices,
            "show_all": True,
        })
        self.assertEqual(ChoiceListFilter, type(filter))
        self.assertEqual('diagnosis_slug', filter.name)
        self.assertEqual('Diagnosis', filter.label)
        self.assertEqual(3, len(filter.choices))
        non_all_choices = filter.choices[1:]
        for i, choice in enumerate(choices):
            self.assertEqual(non_all_choices[i].value, choice['value'])
            self.assertEqual(non_all_choices[i].display, choice['display'])

        # ensure integer values work
        self.assertEqual('positive', filter.value(diagnosis_slug=1).display)
        # ensure 0 integer value works
        self.assertEqual('negative', filter.value(diagnosis_slug=0).display)
        # check string to int coercion
        self.assertEqual('positive', filter.value(diagnosis_slug='1').display)
        # ensure 0 string to int works
        self.assertEqual('negative', filter.value(diagnosis_slug='0').display)

        # check missing values raise errors
        with self.assertRaises(FilterValueException):
            filter.value(diagnosis_slug='4')

        # check non-integers raise errors
        with self.assertRaises(FilterValueException):
            filter.value(diagnosis_slug='foo')

        # check that all still works
        self.assertEqual('Show all', filter.value(diagnosis_slug=SHOW_ALL_CHOICE).display)

class DynamicChoiceListFilterTestCase(SimpleTestCase):

    def setUp(self):
        self.filter_spec = {
            "type": "dynamic_choice_list",
            "slug": "dynoslug",
            "field": "dynofield",
            "display": "Dynamic choice list",
            "show_all": False,
        }

    def test_choice_list_filter(self):
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        self.assertEqual(DynamicChoiceListFilter, type(filter))
        self.assertEqual('dynoslug', filter.name)
        self.assertEqual('Dynamic choice list', filter.label)

    def test_string_datatype(self):
        self.filter_spec['datatype'] = 'string'
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        tests = (
            ('1', '1'),
            ('1.2', '1.2'),
            ('hello', 'hello'),
        )
        for input, expected in tests:
            choices = filter.value(dynoslug=input)
            self.assertEqual(len(choices), 1)
            self.assertEqual(expected, choices[0].value)
            self.assertEqual(input, choices[0].display)

    def test_integer_datatype(self):
        self.filter_spec['datatype'] = 'integer'
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        tests = (
            ('1', 1, '1'),
            ('1.2', 1, '1'),
            ('hello', None, ''),
        )
        for input, expected_value, expected_display in tests:
            choices = filter.value(dynoslug=input)
            self.assertEqual(len(choices), 1)
            self.assertEqual(expected_value, choices[0].value)
            self.assertEqual(expected_display, choices[0].display)

    def test_multiple_selections(self):
        self.filter_spec["datatype"] = "string"
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        test_strings = (
            u'apple',
            u'apple{s}banana'.format(s=CHOICE_DELIMITER),
            u'apple{s}banana{s}carrot'.format(s=CHOICE_DELIMITER)
        )
        choices = [
            Choice('apple', 'apple'),
            Choice('banana', 'banana'),
            Choice('carrot', 'carrot')
        ]
        for i, s in enumerate(test_strings):
            self.assertListEqual(choices[0:i + 1], filter.value(dynoslug=s))

class DateFilterOffsetTest(SimpleTestCase):
    def _computed_dates(self, actual_startdate, actual_enddate):
        filter = ReportFilter(
            compare_as_string=False,
            field=u'submission_date',
            slug=u'submitted_on',
            type=u'date',
            required=False
        )
        value = DateSpan(actual_startdate, actual_enddate)
        filter_value = DateFilterValue(filter, value)
        computed_values = filter_value.to_sql_values()
        startdate = computed_values['%s_startdate' % filter.slug]
        enddate = computed_values['%s_enddate' % filter.slug]
        return startdate, enddate

    def test_date_objects(self):
        start, end = date(2015, 1, 1), date(2015, 1, 2)
        computed_start, computed_end = self._computed_dates(start, end)
        self.assertEqual(computed_start, start)
        self.assertEqual(computed_end, end)

    def test_datetime_objects(self):
        # computed_enddate should be last minute of the enddate
        start, end = datetime(2015, 1, 1), datetime(2015, 1, 2)
        computed_start, computed_end = self._computed_dates(start, end)
        self.assertEqual(computed_start, start)
        self.assertNotEqual(computed_end, end)
        self.assertEqual((computed_end - end).days, 0)
<commit_msg>PreFilterTestCase<commit_after>from datetime import datetime, date
from django.test import SimpleTestCase
from mock import Mock

from corehq.apps.reports_core.exceptions import FilterValueException
from corehq.apps.reports_core.filters import DatespanFilter, ChoiceListFilter, \
    NumericFilter, DynamicChoiceListFilter, Choice, PreFilter
from corehq.apps.userreports.exceptions import BadSpecError
from corehq.apps.userreports.reports.filters.values import SHOW_ALL_CHOICE, \
    CHOICE_DELIMITER, NumericFilterValue, DateFilterValue, PreFilterValue
from corehq.apps.userreports.reports.filters.factory import ReportFilterFactory
from corehq.apps.userreports.reports.filters.specs import ReportFilter
from dimagi.utils.dates import DateSpan

class FilterTestCase(SimpleTestCase):

    def test_no_type(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "field": "some_field",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_bad_type(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "invalid_type",
                "field": "some_field",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_missing_field(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "date",
                "slug": "some_slug",
                "display": "Some display name"
            })

    def test_missing_slug(self):
        with self.assertRaises(BadSpecError):
            ReportFilterFactory.from_spec({
                "type": "date",
                "field": "some_field",
                "display": "Some display name"
            })

    def test_translation(self):
        shared_conf = {
            "type": "date",
            "field": "some_field",
            "slug": "some_slug",
        }

        # Plain string
        conf = {"display": "foo"}
        conf.update(shared_conf)
        filter = ReportFilterFactory.from_spec(conf)
        self.assertEqual(filter.context(None, lang=None)['label'], "foo")
        self.assertEqual(filter.context(None, lang="fr")['label'], "foo")

        # Translation
        conf = {"display": {"en": "english", "fr": "french"}}
        conf.update(shared_conf)
        filter = ReportFilterFactory.from_spec(conf)
        self.assertEqual(filter.context(None, lang=None)['label'], "english")
        self.assertEqual(filter.context(None, lang="fr")['label'], "french")
        self.assertEqual(filter.context(None, lang="en")['label'], "english")
        self.assertEqual(filter.context(None, lang="es")['label'], "english")

class DateFilterTestCase(SimpleTestCase):

    def test_date_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "date",
            "field": "modified_on_field",
            "slug": "modified_on_slug",
            "display": "Date Modified"
        })
        self.assertEqual(DatespanFilter, type(filter))
        self.assertEqual('modified_on_slug', filter.name)
        self.assertEqual('Date Modified', filter.label)

    def test_compare_as_string_option(self):

        def get_query_value(compare_as_string):

            spec = {
                "type": "date",
                "field": "modified_on_field",
                "slug": "my_slug",
                "display": "date Modified",
                "compare_as_string": compare_as_string,
            }
            reports_core_filter = ReportFilterFactory.from_spec(spec)
            reports_core_value = reports_core_filter.get_value({
                "my_slug-start": "2015-06-07",
                "my_slug-end": "2015-06-08",
                "date_range_inclusive": True,
            })

            filter = ReportFilter.wrap(spec)
            return filter.create_filter_value(reports_core_value).to_sql_values()

        val = get_query_value(compare_as_string=False)
        self.assertEqual(type(val['my_slug_startdate']), datetime)
        self.assertEqual(type(val['my_slug_enddate']), datetime)

        val = get_query_value(compare_as_string=True)
        self.assertEqual(type(val['my_slug_startdate']), str)
        self.assertEqual(type(val['my_slug_enddate']), str)

class NumericFilterTestCase(SimpleTestCase):

    def test_numeric_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "numeric",
            "field": "number_of_children_field",
            "slug": "number_of_children_slug",
            "display": "Number of Children",
        })
        self.assertEqual(NumericFilter, type(filter))
        self.assertEqual("number_of_children_slug", filter.name)
        self.assertEqual("Number of Children", filter.label)

    def test_numeric_filter_value(self):
        filter = ReportFilter.wrap({
            "type": "numeric",
            "field": "number_of_children_field",
            "slug": "number_of_children_slug",
            "display": "Number of Children",
        })
        NumericFilterValue(filter, None)
        NumericFilterValue(filter, {'operator': '<', 'operand': 3})
        with self.assertRaises(AssertionError):
            NumericFilterValue(filter, {'operator': 'sql injection', 'operand': 3})

class PreFilterTestCase(SimpleTestCase):

    def test_auto_filter(self):
        filter_ = ReportFilterFactory.from_spec({
            'type': 'pre',
            'field': 'at_risk_field',
            'slug': 'at_risk_slug',
            'datatype': 'string',
            'pre_value': 'true'
        })
        self.assertEqual(type(filter_), PreFilter)
        self.assertEqual(filter_.name, 'at_risk_slug')
        self.assertEqual(filter_.default_value(), 'true')

    def test_auto_filter_value(self):
        pre_value = 'yes'
        filter_ = ReportFilter.wrap({
            'type': 'pre',
            'field': 'at_risk_field',
            'slug': 'at_risk_slug',
            'datatype': 'string',
            'pre_value': pre_value
        })
        value = PreFilterValue(filter_, pre_value)
        self.assertEqual(value.to_sql_values(), {'at_risk_slug': 'yes'})

    def test_auto_filter_value_null(self):
        column = Mock()
        column.name = 'at_risk_field'
        column.is_.return_value = 'foo'
        table = Mock()
        table.c = [column]

        pre_value = None
        filter_ = ReportFilter.wrap({
            'type': 'pre',
            'field': 'at_risk_field',
            'slug': 'at_risk_slug',
            'datatype': 'string',
            'pre_value': pre_value
        })
        value = PreFilterValue(filter_, pre_value)
        self.assertEqual(value.to_sql_values(), {})
        self.assertEqual(value.to_sql_filter().build_expression(table), 'foo')

    def test_auto_filter_value_array(self):
        column = Mock()
        column.name = 'at_risk_field'
        column.in_.return_value = 'foo'
        table = Mock()
        table.c = [column]

        pre_value = ['yes', 'maybe']
        filter_ = ReportFilter.wrap({
            'type': 'pre',
            'field': 'at_risk_field',
            'slug': 'at_risk_slug',
            'datatype': 'array',
            'pre_value': pre_value
        })
        value = PreFilterValue(filter_, pre_value)
        self.assertEqual(value.to_sql_values(), {'at_risk_slug_0': 'yes', 'at_risk_slug_1': 'maybe'})
        self.assertEqual(value.to_sql_filter().build_expression(table), 'foo')

class ChoiceListFilterTestCase(SimpleTestCase):
    CHOICES = [
        {
            "value": "NEGATIVE",
            "display": "negative"
        },
        {
            "value": "POSITIVE",
            "display": "positive"
        }
    ]

    def test_choice_list_filter(self):
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "display": "Diagnosis",
            "choices": self.CHOICES,
            "show_all": False,
        })
        self.assertEqual(ChoiceListFilter, type(filter))
        self.assertEqual('diagnosis_slug', filter.name)
        self.assertEqual('Diagnosis', filter.label)
        self.assertEqual(2, len(filter.choices))
        for i, choice in enumerate(self.CHOICES):
            self.assertEqual(filter.choices[i].value, choice['value'])
            self.assertEqual(filter.choices[i].display, choice['display'])

        # check values
        self.assertEqual('positive', filter.value(diagnosis_slug='POSITIVE').display)

    def test_choice_list_filter_show_all(self):
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "display": "Diagnosis",
            "choices": self.CHOICES,
            "show_all": True,
        })
        self.assertEqual(3, len(filter.choices))

        self.assertEqual(SHOW_ALL_CHOICE, filter.choices[0].value)
        for i, choice in enumerate(self.CHOICES):
            self.assertEqual(filter.choices[i + 1].value, choice['value'])
            self.assertEqual(filter.choices[i + 1].display, choice['display'])

        # check all value
        self.assertEqual('Show all', filter.value(diagnosis_slug=SHOW_ALL_CHOICE).display)

    def test_choice_list_filter_with_integers(self):
        choices = [
            {
                "value": 0,
                "display": "negative"
            },
            {
                "value": 1,
                "display": "positive"
            }
        ]
        filter = ReportFilterFactory.from_spec({
            "type": "choice_list",
            "slug": "diagnosis_slug",
            "field": "diagnosis_field",
            "datatype": "integer",
            "display": "Diagnosis",
            "choices": choices,
            "show_all": True,
        })
        self.assertEqual(ChoiceListFilter, type(filter))
        self.assertEqual('diagnosis_slug', filter.name)
        self.assertEqual('Diagnosis', filter.label)
        self.assertEqual(3, len(filter.choices))
        non_all_choices = filter.choices[1:]
        for i, choice in enumerate(choices):
            self.assertEqual(non_all_choices[i].value, choice['value'])
            self.assertEqual(non_all_choices[i].display, choice['display'])

        # ensure integer values work
        self.assertEqual('positive', filter.value(diagnosis_slug=1).display)
        # ensure 0 integer value works
        self.assertEqual('negative', filter.value(diagnosis_slug=0).display)
        # check string to int coercion
        self.assertEqual('positive', filter.value(diagnosis_slug='1').display)
        # ensure 0 string to int works
        self.assertEqual('negative', filter.value(diagnosis_slug='0').display)

        # check missing values raise errors
        with self.assertRaises(FilterValueException):
            filter.value(diagnosis_slug='4')

        # check non-integers raise errors
        with self.assertRaises(FilterValueException):
            filter.value(diagnosis_slug='foo')

        # check that all still works
        self.assertEqual('Show all', filter.value(diagnosis_slug=SHOW_ALL_CHOICE).display)

class DynamicChoiceListFilterTestCase(SimpleTestCase):

    def setUp(self):
        self.filter_spec = {
            "type": "dynamic_choice_list",
            "slug": "dynoslug",
            "field": "dynofield",
            "display": "Dynamic choice list",
            "show_all": False,
        }

    def test_choice_list_filter(self):
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        self.assertEqual(DynamicChoiceListFilter, type(filter))
        self.assertEqual('dynoslug', filter.name)
        self.assertEqual('Dynamic choice list', filter.label)

    def test_string_datatype(self):
        self.filter_spec['datatype'] = 'string'
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        tests = (
            ('1', '1'),
            ('1.2', '1.2'),
            ('hello', 'hello'),
        )
        for input, expected in tests:
            choices = filter.value(dynoslug=input)
            self.assertEqual(len(choices), 1)
            self.assertEqual(expected, choices[0].value)
            self.assertEqual(input, choices[0].display)

    def test_integer_datatype(self):
        self.filter_spec['datatype'] = 'integer'
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        tests = (
            ('1', 1, '1'),
            ('1.2', 1, '1'),
            ('hello', None, ''),
        )
        for input, expected_value, expected_display in tests:
            choices = filter.value(dynoslug=input)
            self.assertEqual(len(choices), 1)
            self.assertEqual(expected_value, choices[0].value)
            self.assertEqual(expected_display, choices[0].display)

    def test_multiple_selections(self):
        self.filter_spec["datatype"] = "string"
        filter = ReportFilterFactory.from_spec(self.filter_spec)
        test_strings = (
            u'apple',
            u'apple{s}banana'.format(s=CHOICE_DELIMITER),
            u'apple{s}banana{s}carrot'.format(s=CHOICE_DELIMITER)
        )
        choices = [
            Choice('apple', 'apple'),
            Choice('banana', 'banana'),
            Choice('carrot', 'carrot')
        ]
        for i, s in enumerate(test_strings):
            self.assertListEqual(choices[0:i + 1], filter.value(dynoslug=s))

class DateFilterOffsetTest(SimpleTestCase):
    def _computed_dates(self, actual_startdate, actual_enddate):
        filter = ReportFilter(
            compare_as_string=False,
            field=u'submission_date',
            slug=u'submitted_on',
            type=u'date',
            required=False
        )
        value = DateSpan(actual_startdate, actual_enddate)
        filter_value = DateFilterValue(filter, value)
        computed_values = filter_value.to_sql_values()
        startdate = computed_values['%s_startdate' % filter.slug]
        enddate = computed_values['%s_enddate' % filter.slug]
        return startdate, enddate

    def test_date_objects(self):
        start, end = date(2015, 1, 1), date(2015, 1, 2)
        computed_start, computed_end = self._computed_dates(start, end)
        self.assertEqual(computed_start, start)
        self.assertEqual(computed_end, end)

    def test_datetime_objects(self):
        # computed_enddate should be last minute of the enddate
        start, end = datetime(2015, 1, 1), datetime(2015, 1, 2)
        computed_start, computed_end = self._computed_dates(start, end)
        self.assertEqual(computed_start, start)
        self.assertNotEqual(computed_end, end)
        self.assertEqual((computed_end - end).days, 0)
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime

from django.test import TestCase

from corehq.apps.export.models import ExportInstance
from corehq.apps.export.models.new import DAILY_SAVED_EXPORT_ATTACHMENT_NAME
from couchexport.groupexports import get_saved_export_and_delete_copies
from couchexport.models import SavedBasicExport, ExportConfiguration
from six.moves import range

class DailySavedExportSavingTest(TestCase):

    def test_file_save_and_load(self):
        payload = 'something small and simple'
        export = ExportInstance(daily_saved_export=True, domain="test")
        export.save()
        export.set_payload(payload)
        self.assertEqual(payload, export.get_payload())

    def test_save_basic_export_to_blobdb(self):
        export = ExportInstance(daily_saved_export=True, domain="test")
        export.save()
        export.set_payload("content")
        self.assertTrue(export.has_file())
        self.assertIn(DAILY_SAVED_EXPORT_ATTACHMENT_NAME, export.external_blobs)
        self.assertEqual(export.file_size, 7)
        with export.get_payload(stream=True) as fh:
            self.assertEqual(fh.read(), "content")

class SavedBasicExportTest(TestCase):

    def test_file_save_and_load(self):
        payload = 'something small and simple'
        for name in ['normal', 'हिंदी', None]:
            saved = SavedBasicExport(configuration=_mk_config(name))
            saved.save()
            saved.set_payload(payload)
            self.assertEqual(payload, saved.get_payload())

    def test_get_by_index(self):
        index = ['some', 'index']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        back = SavedBasicExport.by_index(index)
        self.assertEqual(1, len(back))
        self.assertEqual(saved_export._id, back[0]._id)

    def test_get_saved_and_delete_copies_missing(self):
        self.assertEqual(None, get_saved_export_and_delete_copies(['missing', 'index']))

    def test_get_saved_and_delete_copies_single(self):
        index = ['single']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        self.assertEqual(saved_export._id, get_saved_export_and_delete_copies(index)._id)

    def test_get_saved_and_delete_copies_multiple(self):
        index = ['multiple']
        # make three exports with the last one being the most recently updated
        timestamp = datetime.datetime.utcnow()
        for i in range(3):
            saved_export = SavedBasicExport(configuration=_mk_config(index=index),
                                            last_updated=timestamp + datetime.timedelta(days=i))
            saved_export.save()

        self.assertEqual(3, len(SavedBasicExport.by_index(index)))
        chosen_one = get_saved_export_and_delete_copies(index)
        # this relies on the variable being set last in the loop which is a bit unintuitive
        self.assertEqual(saved_export._id, chosen_one._id)
        saved_after_deletion = SavedBasicExport.by_index(index)
        self.assertEqual(1, len(saved_after_deletion))
        self.assertEqual(chosen_one._id, saved_after_deletion[0]._id)

    def test_save_basic_export_to_blobdb(self):
        index = ['single']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        saved_export.set_payload("content")
        name = saved_export.get_attachment_name()
        self.assertTrue(saved_export.has_file())
        self.assertIn(name, saved_export.external_blobs)
        self.assertEqual(saved_export.size, 7)
        with saved_export.get_payload(stream=True) as fh:
            self.assertEqual(fh.read(), b"content")

def _mk_config(name='some export name', index='dummy_index'):
    return ExportConfiguration(index=index, name=name, format='xlsx')
<commit_msg>fix couchexport.tests.test_saved:DailySavedExportSavingTest.test_save_basic_export_to_blobdb<commit_after># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime

from django.test import TestCase

from corehq.apps.export.models import ExportInstance
from corehq.apps.export.models.new import DAILY_SAVED_EXPORT_ATTACHMENT_NAME
from couchexport.groupexports import get_saved_export_and_delete_copies
from couchexport.models import SavedBasicExport, ExportConfiguration
from six.moves import range

class DailySavedExportSavingTest(TestCase):

    def test_file_save_and_load(self):
        payload = 'something small and simple'
        export = ExportInstance(daily_saved_export=True, domain="test")
        export.save()
        export.set_payload(payload)
        self.assertEqual(payload, export.get_payload())

    def test_save_basic_export_to_blobdb(self):
        export = ExportInstance(daily_saved_export=True, domain="test")
        export.save()
        export.set_payload("content")
        self.assertTrue(export.has_file())
        self.assertIn(DAILY_SAVED_EXPORT_ATTACHMENT_NAME, export.external_blobs)
        self.assertEqual(export.file_size, 7)
        with export.get_payload(stream=True) as fh:
            self.assertEqual(fh.read(), b"content")

class SavedBasicExportTest(TestCase):

    def test_file_save_and_load(self):
        payload = 'something small and simple'
        for name in ['normal', 'हिंदी', None]:
            saved = SavedBasicExport(configuration=_mk_config(name))
            saved.save()
            saved.set_payload(payload)
            self.assertEqual(payload, saved.get_payload())

    def test_get_by_index(self):
        index = ['some', 'index']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        back = SavedBasicExport.by_index(index)
        self.assertEqual(1, len(back))
        self.assertEqual(saved_export._id, back[0]._id)

    def test_get_saved_and_delete_copies_missing(self):
        self.assertEqual(None, get_saved_export_and_delete_copies(['missing', 'index']))

    def test_get_saved_and_delete_copies_single(self):
        index = ['single']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        self.assertEqual(saved_export._id, get_saved_export_and_delete_copies(index)._id)

    def test_get_saved_and_delete_copies_multiple(self):
        index = ['multiple']
        # make three exports with the last one being the most recently updated
        timestamp = datetime.datetime.utcnow()
        for i in range(3):
            saved_export = SavedBasicExport(configuration=_mk_config(index=index),
                                            last_updated=timestamp + datetime.timedelta(days=i))
            saved_export.save()

        self.assertEqual(3, len(SavedBasicExport.by_index(index)))
        chosen_one = get_saved_export_and_delete_copies(index)
        # this relies on the variable being set last in the loop which is a bit unintuitive
        self.assertEqual(saved_export._id, chosen_one._id)
        saved_after_deletion = SavedBasicExport.by_index(index)
        self.assertEqual(1, len(saved_after_deletion))
        self.assertEqual(chosen_one._id, saved_after_deletion[0]._id)

    def test_save_basic_export_to_blobdb(self):
        index = ['single']
        saved_export = SavedBasicExport(configuration=_mk_config(index=index))
        saved_export.save()
        saved_export.set_payload("content")
        name = saved_export.get_attachment_name()
        self.assertTrue(saved_export.has_file())
        self.assertIn(name, saved_export.external_blobs)
        self.assertEqual(saved_export.size, 7)
        with saved_export.get_payload(stream=True) as fh:
            self.assertEqual(fh.read(), b"content")

def _mk_config(name='some export name', index='dummy_index'):
    return ExportConfiguration(index=index, name=name, format='xlsx')
<|endoftext|>