<commit_before>import glob, os, re, traceback
from importlib.machinery import SourceFileLoader
from multiprocessing import Pool
from django import db
from django.apps import apps
from django.core.management.base import BaseCommand

from pollingstations.models import PollingStation

# does this regular expression match any of the elements in this list?
def match_in(regex, lst):
    for el in lst:
        if re.match(regex, el):
            return True
    return False

# load a django management command from file f
def load_command(f):
    command = SourceFileLoader("module.name", f).load_module()
    return command.Command()

# run a django management command from file f
def run_cmd(f, opts):
    cmd = load_command(f)
    try:
        cmd.handle(**opts)
    except Exception as e:
        traceback.print_exc()
        raise e

"""
Run all of the import scripts relating to a particular election or elections

Election id may be either a string or regex. For example:
python manage.py import -e local.buckinghamshire.2017-05-04
python manage.py import -r -e 'local.[a-z]+.2017-05-04'
"""
class Command(BaseCommand):

    """
    Turn off auto system check for all apps
    We will maunally run system checks only for the
    'data_collection' and 'pollingstations' apps
    """
    requires_system_checks = False

    summary = []

    def add_arguments(self, parser):
        parser.add_argument(
            '-e',
            '--elections',
            nargs='+',
            help='<Required> List of one or more election ids to import data for',
            required=True
        )

        parser.add_argument(
            '-r',
            '--regex',
            help='<Optional> Election ids should be matched as regular expressions',
            action='store_true',
            required=False,
            default=False
        )

        parser.add_argument(
            '-o',
            '--overwrite',
            help='<Optional> Force deleting of existing data',
            action='store_true',
            required=False,
            default=False
        )

        parser.add_argument(
            '-m',
            '--multiprocessing',
            help='<Optional> Use multiprocessing for import',
            action='store_true',
            required=False,
            default=False
        )

    def importer_covers_these_elections(self, args_elections, importer_elections, regex):
        for election in args_elections:
            if regex:
                if match_in(election, importer_elections):
                    return True
            else:
                if election in importer_elections:
                    return True
        return False

    def output_summary(self):
        for line in self.summary:
            if line[0] == 'INFO':
                self.stdout.write(line[1])
            elif line[0] == 'WARNING':
                self.stdout.write(self.style.ERROR(line[1]))
            else:
                self.stdout.write(line[1])

    def run_commands_in_series(self, commands):
        for f, opts in commands:
            run_cmd(f, opts)

    def run_commands_in_parallel(self, commands):
        pool = Pool()
        pool.starmap(run_cmd, commands)
        pool.close()
        pool.join()

    def handle(self, *args, **kwargs):
        """
        Manually run system checks for the
        'data_collection' and 'pollingstations' apps
        Management commands can ignore checks that only apply to
        the apps supporting the website part of the project
        """
        self.check([
            apps.get_app_config('data_collection'),
            apps.get_app_config('pollingstations')
        ])

        base_path = os.path.dirname(__file__)
        files = glob.glob(base_path + '/import_*.py')

        if not files:
            raise ValueError("No importers matched")

        commands_to_run = []
        opts = {'noclean': False, 'verbosity': 1}
        if kwargs['multiprocessing']:
            opts = {'noclean': False, 'verbosity': 0}

        # loop over all the import scripts
        # and build up a list of management commands to run
        for f in files:
            head, tail = os.path.split(f)
            try:
                cmd = load_command(f)
            except:
                # usually we want to handle a specific exception, but in in this situation
                # if there is any issue (at all) trying to load the module,
                # we just want to log it and move on to the next script
                self.summary.append(('WARNING', "%s could not be loaded!" % tail))
                continue

            if hasattr(cmd, 'elections'):
                if self.importer_covers_these_elections(kwargs['elections'], cmd.elections, kwargs['regex']):
                    # Only run if
                    existing_data = PollingStation.objects.filter(
                        council_id=cmd.council_id).exists()
                    if not existing_data or kwargs.get('overwrite'):
                        self.summary.append(
                            ('INFO', "Ran import script %s" % tail))
                        commands_to_run.append((f, opts))
            else:
                self.summary.append(('WARNING', "%s does not contain elections property!" % tail))

        # run the list of import scripts in commands_to_run[]
        if kwargs['multiprocessing']:
            # before kicking off parallel imports, close any open
            # DB connections. Otherwise, Django will throw
            # django.db.utils.DatabaseError: lost synchronization with server
            db.connections.close_all()
            self.run_commands_in_parallel(commands_to_run)
        else:
            self.run_commands_in_series(commands_to_run)

        self.output_summary()
<commit_msg>allow us to specify some commands should not be run in parallel<commit_after>import glob, os, re, traceback
from importlib.machinery import SourceFileLoader
from multiprocessing import Pool
from django import db
from django.apps import apps
from django.core.management.base import BaseCommand

from pollingstations.models import PollingStation

# does this regular expression match any of the elements in this list?
def match_in(regex, lst):
    for el in lst:
        if re.match(regex, el):
            return True
    return False

# load a django management command from file f
def load_command(f):
    command = SourceFileLoader("module.name", f).load_module()
    return command.Command()

# run a django management command from file f
def run_cmd(f, opts):
    cmd = load_command(f)
    try:
        cmd.handle(**opts)
    except Exception as e:
        traceback.print_exc()
        raise e

"""
Run all of the import scripts relating to a particular election or elections

Election id may be either a string or regex. For example:
python manage.py import -e local.buckinghamshire.2017-05-04
python manage.py import -r -e 'local.[a-z]+.2017-05-04'
"""
class Command(BaseCommand):

    """
    Turn off auto system check for all apps
    We will maunally run system checks only for the
    'data_collection' and 'pollingstations' apps
    """
    requires_system_checks = False

    summary = []

    def add_arguments(self, parser):
        parser.add_argument(
            '-e',
            '--elections',
            nargs='+',
            help='<Required> List of one or more election ids to import data for',
            required=True
        )

        parser.add_argument(
            '-r',
            '--regex',
            help='<Optional> Election ids should be matched as regular expressions',
            action='store_true',
            required=False,
            default=False
        )

        parser.add_argument(
            '-o',
            '--overwrite',
            help='<Optional> Force deleting of existing data',
            action='store_true',
            required=False,
            default=False
        )

        parser.add_argument(
            '-m',
            '--multiprocessing',
            help='<Optional> Use multiprocessing for import',
            action='store_true',
            required=False,
            default=False
        )

    def importer_covers_these_elections(self, args_elections, importer_elections, regex):
        for election in args_elections:
            if regex:
                if match_in(election, importer_elections):
                    return True
            else:
                if election in importer_elections:
                    return True
        return False

    def output_summary(self):
        for line in self.summary:
            if line[0] == 'INFO':
                self.stdout.write(line[1])
            elif line[0] == 'WARNING':
                self.stdout.write(self.style.ERROR(line[1]))
            else:
                self.stdout.write(line[1])

    def run_commands_in_series(self, commands):
        for f, opts in commands:
            run_cmd(f, opts)

    def run_commands_in_parallel(self, commands):
        pool = Pool()
        pool.starmap(run_cmd, commands)
        pool.close()
        pool.join()

    def handle(self, *args, **kwargs):
        """
        Manually run system checks for the
        'data_collection' and 'pollingstations' apps
        Management commands can ignore checks that only apply to
        the apps supporting the website part of the project
        """
        self.check([
            apps.get_app_config('data_collection'),
            apps.get_app_config('pollingstations')
        ])

        base_path = os.path.dirname(__file__)
        files = glob.glob(base_path + '/import_*.py')

        if not files:
            raise ValueError("No importers matched")

        commands_series = []
        commands_parallel = []
        opts = {'noclean': False, 'verbosity': 1}
        if kwargs['multiprocessing']:
            opts = {'noclean': False, 'verbosity': 0}

        # loop over all the import scripts
        # and build up a list of management commands to run
        for f in files:
            head, tail = os.path.split(f)
            try:
                cmd = load_command(f)
            except:
                # usually we want to handle a specific exception, but in in this situation
                # if there is any issue (at all) trying to load the module,
                # we just want to log it and move on to the next script
                self.summary.append(('WARNING', "%s could not be loaded!" % tail))
                continue

            if hasattr(cmd, 'elections'):
                if self.importer_covers_these_elections(kwargs['elections'], cmd.elections, kwargs['regex']):
                    # Only run if
                    existing_data = PollingStation.objects.filter(
                        council_id=cmd.council_id).exists()
                    if not existing_data or kwargs.get('overwrite'):
                        self.summary.append(
                            ('INFO', "Ran import script %s" % tail))
                        if hasattr(cmd, 'run_in_series'):
                            commands_series.append((f, opts))
                        else:
                            commands_parallel.append((f, opts))
            else:
                self.summary.append(('WARNING', "%s does not contain elections property!" % tail))

        # run all the import scripts
        if kwargs['multiprocessing']:
            # do anything we want to run in series first
            self.run_commands_in_series(commands_series)

            # before kicking off parallel imports, close any open
            # DB connections. Otherwise, Django will throw
            # django.db.utils.DatabaseError: lost synchronization with server
            db.connections.close_all()
            self.run_commands_in_parallel(commands_parallel)
        else:
            self.run_commands_in_series(commands_parallel + commands_series)

        self.output_summary()
<|endoftext|>


<commit_before>__version__ = '0.4'
__author__ = 'Decklin Foster'
__email__ = 'decklin@red-bean.com'
__url__ = 'http://www.red-bean.com/~decklin/software/mnemosyne/'

import os
import locale
import mailbox
import email, email.Message, email.Header
import time
import stat
import shutil
import em
from rsthtml import publish_content

class Muse:
    def __init__(self, config, force):
        self.force = force
        self.where = []

        default_dir = os.path.join(os.environ['HOME'], 'Mnemosyne')
        self.conf = {
            'entry_dir': os.path.join(default_dir, 'entries'),
            'layout_dir': os.path.join(default_dir, 'layout'),
            'style_dir': os.path.join(default_dir, 'style'),
            'output_dir': os.path.join(default_dir, 'htdocs'),
            'ignore': ('.svn', 'CVS'),
            'charset': None,
            'locals': {
                '__version__': __version__,
                '__author__': __author__,
                '__email__': __email__,
                '__url__': __url__,
                },
            }

        for d in ('entry_dir', 'layout_dir', 'style_dir', 'output_dir'):
            if not os.path.exists(self.conf[d]):
                raise RuntimeError("%s %s does not exist" % (d, self.conf[d]))

        try:
            exec file(config) in self.conf
        except Exception, e:
            raise RuntimeError("Error running config: %s" % e)

        try: Entry.__bases__ += (self.conf['EntryMixin'],)
        except KeyError: pass

        method = lambda _self, obj, rep: magic(obj, rep, self.conf['charset'])
        Entry.magic = method

        # It would be nice if the factory could decide whether it needs to
        # open the file or not. All we need to know here is mtime, and all we
        # need for that is the filename/inode itself. But either way we need
        # to grab everything from the iterator up front so we can sort.

        self.box = mailbox.Maildir(self.conf['entry_dir'], Entry)
        self.entries = [e for e in self.box]
        self.entries.sort()

    def sing(self, entries=None, spath=None, dpath=None, what=None):
        """From the contents of spath, build output in dpath, based on the
        provided entries. For each entry in spath, will be called recursively
        with a tuple what representing the source and dest file. For any
        source files starting with __attr__ will recur several times based on
        which entries match each value of that attribute. For regularly named
        files, evaluate them as layout scripts if they are executable and
        simply copy them if they are not."""

        if not entries: entries = self.entries
        if not spath: spath = self.conf['layout_dir']
        if not dpath: dpath = self.conf['output_dir']

        def stale(dpath, spath, entries=None):
            """Test if the file named by dpath is nonexistent or older than
            either the file named by spath or any entry in the given list of
            entries. If --force has been turned on, always return True."""

            if self.force or not os.path.exists(dpath):
                return True
            else:
                dmtime = os.path.getmtime(dpath)
                smtimes = [os.path.getmtime(spath)]
                if entries: smtimes += [time.mktime(e.mtime) for e in entries]
                return dmtime < max(smtimes)

        if what:
            source, dest = what
            spath = os.path.join(spath, source)
            dpath = os.path.join(dpath, dest)
            if source not in self.conf['ignore']:
                if os.path.isfile(spath):
                    if os.stat(spath).st_mode & stat.S_IXUSR:
                        if stale(dpath, spath, entries):
                            self.sing_file(entries, spath, dpath)
                    else:
                        if stale(dpath, spath):
                            print 'Copied %s' % dpath
                            shutil.copyfile(spath, dpath)
                elif os.path.isdir(spath):
                    self.sing(entries, spath, dpath)
        else:
            if not os.path.isdir(dpath): os.makedirs(dpath)
            for f in os.listdir(spath):
                if f.startswith('__'):
                    self.sing_instances(entries, spath, dpath, f)
                else:
                    self.where.append(f)
                    self.sing(entries, spath, dpath, (f, f))
                    self.where.pop()

    def sing_instances(self, entries, spath, dpath, what):
        """Given a source and dest file in the tuple what, where the source
        starts with __attr__, group the provided entries by the values of that
        attribute over all the provided entries. For an entry e and attribute
        attr, e.attr may be an atomic value or a sequence of values. For each
        value so encountered, evaluate the source file given all entries in
        entries that match that value."""

        subst = what[:what.rindex('__')+2]

        def cheapiter(x):
            """DWIM-style iterator which, if given a sequence, will iterate
            over that sequence, unless it is a string type. For a string or
            any other atomic type, create an iterator which will return the
            given value once and then stop. This is a horrible kludge."""

            try:
                assert not isinstance((x), (str, unicode))
                for e in x: return iter(x)
                else: return iter(())
            except (AssertionError, TypeError):
                if x: return iter((x,))
                else: return iter(())

        inst = {}
        for e in entries:
            mv = getattr(e, subst[2:-2])
            for m in cheapiter(mv):
                inst.setdefault(repr(m), []).append(e)

        for k, entries in inst.iteritems():
            self.where.append(k)
            self.sing(entries, spath, dpath, (what, what.replace(subst, k)))
            self.where.pop()

    def expand(self, style, locals):
        """Open an EmPy file in the configuration's style directory, and
        evaluate it with the given locals."""
        style = os.path.join(self.conf['style_dir'], '%s.empy' % style)
        try:
            return em.expand(file(style).read(), locals)
        except Exception, e:
            raise RuntimeError("Error running style %s: %s" % (style, e))

    def sing_file(self, entries, spath, dpath):
        """Given an actual source and dest file, where the source is a layout
        script, evaluate it given the locals from config plus muse (ourself),
        write (a callback which actually writes the file), and entries (the
        given entries)."""

        def write(data):
            file(dpath, 'w').write(data)
            print 'Wrote %s' % dpath

        locals = self.conf['locals'].copy()
        locals['muse'] = self
        locals['write'] = write
        locals['entries'] = entries

        try:
            exec file(spath) in locals
        except Exception, e:
            raise RuntimeError("Error running layout %s: %s" % (spath, e))

class BaseEntry:
    """Base class for all entries. Initialized with an open file object, so it
    may be passed to maildir.Maildir as a factory class. Parses the file's
    contents as a Message object, setting a date attribute from the parsed
    date and an mtime attribute from the Maildir filename."""

    def __init__(self, fp):
        def fixdate(d):
            # For some bizarre reason, parsedate doesn't set wday/yday/isdst.
            return time.localtime(time.mktime(d))
        def getstamp(mpath):
            # djb says you're not supposed to do this. djb can bite me.
            stamp, id, host = os.path.split(mpath)[1].split('.')
            return int(stamp)

        self.msg = email.message_from_file(fp, Message)
        self.date = fixdate(email.Utils.parsedate(self.msg['Date']))
        self.mtime = time.localtime(getstamp(fp.name))

    def __cmp__(self, other):
        if other:
            return cmp(time.mktime(self.date), time.mktime(other.date))
        else:
            return 1

    rendered = {}
    def _prop_content(self):
        """Read in the message's body, strip any signature, and format using
        reStructedText."""

        try:
            return self.rendered[hash(self.msg)]
        except KeyError:
            s = self.msg.get_payload(decode=True)
            if not s: return ''

            try: s = s[:s.rindex('-- \n')]
            except ValueError: pass

            html = self.magic(publish_content(s), s[:100])
            return self.rendered.setdefault(hash(self.msg), html)

    byday = {}
    def _init_subject(self):
        """Provide the contents of the Subject: header and a cleaned, uniq'd
        version of same."""

        try:
            subject = self.msg['Subject']
            cleaned = clean(subject, 3)
        except KeyError:
            subject = ''
            cleaned = 'entry'

        # Grab the namespace for the day of this entry
        day = self.byday.setdefault(self.date[0:3], UniqueDict())

        slug = day.setdefault(hash(self.msg), cleaned)
        return self.magic(subject, slug)

    def _init_id(self):
        """Provide the Message-ID and a globally unique tag: URL based on it,
        for use in feeds."""

        try:
            id = self.msg['Message-Id'][1:-1]
            local, host = id.split('@')
            date = time.strftime('%Y-%m-%d', self.date)
            return self.magic(id, 'tag:%s,%s:%s' % (host, date, local))
        except KeyError:
            return ''

    def _init_author(self):
        author, addr = email.Utils.parseaddr(self.msg.get('From'))
        return self.magic(author, clean(author))

    def _init_email(self):
        """Provide the author's email address and a trivially spam-protected
        version of same."""
        try:
            author, addr = email.Utils.parseaddr(self.msg['From'])
            cleaned = addr.replace('@', ' at ')
            cleaned = cleaned.replace('.', ' dot ')
            cleaned = cleaned.replace('-', ' dash ')
            return self.magic(addr, cleaned)
        except KeyError:
            return ''

    def _init_tags(self):
        """Provide a list of tags from the comma-delimited X-Tags: header."""
        try:
            tags = [t.strip() for t in self.msg['X-Tags'].split(',')]
            return [self.magic(t, clean(t)) for t in tags]
        except KeyError:
            return []

    def _init_year(self):
        return self.magic(self.date[0], time.strftime('%Y', self.date))

    def _init_month(self):
        return self.magic(self.date[1], time.strftime('%m', self.date))

    def _init_day(self):
        return self.magic(self.date[2], time.strftime('%d', self.date))

class Entry(BaseEntry):
    """Actual entry class. Will search the user-provided mixin class and then
    BaseEntry for methods of the form _init_*, and set the appropriate
    attribute on initialization, and also search for methods of the form
    _prop_* to provide properties to be evaluated on demand."""

    def __init__(self, fp):
        for _class in self.__class__.__bases__:
            try: _class.__init__(self, fp)
            except AttributeError: pass

            for k, v in _class.__dict__.iteritems():
                if k.startswith('_init_'):
                    setattr(self, k[6:], v(self))
                if k.startswith('_prop_'):
                    setattr(self.__class__, k[6:], property(v, None))

class Message(email.Message.Message):
    """Non-broken version of email's Message class. Returns unicode headers
    when necessary and raises KeyError."""

    def __getitem__(self, item):
        header = email.Message.Message.__getitem__(self, item)
        if header:
            # worst. interface. ever.
            parts = email.Header.decode_header(header)
            decoded = []
            for p, enc in parts:
                if enc:
                    decoded.append(unicode(p, enc))
                else:
                    decoded.append(p)
            return ' '.join(decoded)
        else:
            raise KeyError

class UniqueDict(dict):
    """A read-only dict which munges its values so that they are unique. If an
    existing key has the value 'foo', attempting to set another key to 'foo'
    will cause it to become 'foo-1', then 'foo-2', etc. These numberings are
    stable as long as each key is assigned to in the same order; attempting to
    set an existing key will cause a ValueError. """

    def __getitem__(self, k):
        k, i = dict.__getitem__(self, k)
        if i: return '%s-%d' % (k, i)
        else: return k

    def __setitem__(self, k, v):
        if k in self: raise ValueError
        n = len([x for x, y in self.iteritems() if y[0] == v])
        dict.__setitem__(self, k, (v, n))

    # Yes, we must. Le sigh.
    def setdefault(self, key, failobj=None):
        if not self.has_key(key): self[key] = failobj
        return self[key]

def magic(obj, rep, enc):
    """Make obj into something suitable for passing to a layout. Returns an
    object exactly like obj, except its repr() is rep and both are encoded if
    they were unicode (layouts must serialize as a str, so giving them unicode
    data is not recommended). If you are not passing something that might be a
    unicode string, and do not care about using its repr() in your layout, you
    are not required to use this function."""

    if type(obj) is unicode: obj = obj.encode(enc)
    if type(rep) is unicode: rep = rep.encode(enc)

    _class = type("Magic", (type(obj),), {'__repr__': lambda self: rep})
    return _class(obj)

def clean(s, maxwords=None):
    """Split the given string into words, lowercase and strip all
    non-alphanumerics from them, and join them with '-'. If maxwords is given,
    limit the returned string to that many words. If the string is None,
    return None."""

    try:
        words = s.strip().lower().split()[:maxwords]
        words = [''.join([c for c in w if c.isalnum()]) for w in words]
        return '-'.join(words)
    except AttributeError:
        return None
<commit_msg>I got yer docstrings right here, buddy.<commit_after>__version__ = '0.4'
__author__ = 'Decklin Foster'
__email__ = 'decklin@red-bean.com'
__url__ = 'http://www.red-bean.com/~decklin/software/mnemosyne/'

import os
import locale
import mailbox
import email, email.Message, email.Header
import time
import stat
import shutil
import em
from rsthtml import publish_content

class Muse:
    def __init__(self, config, force):
        self.force = force
        self.where = []

        default_dir = os.path.join(os.environ['HOME'], 'Mnemosyne')
        self.conf = {
            'entry_dir': os.path.join(default_dir, 'entries'),
            'layout_dir': os.path.join(default_dir, 'layout'),
            'style_dir': os.path.join(default_dir, 'style'),
            'output_dir': os.path.join(default_dir, 'htdocs'),
            'ignore': ('.svn', 'CVS'),
            'charset': None,
            'locals': {
                '__version__': __version__,
                '__author__': __author__,
                '__email__': __email__,
                '__url__': __url__,
                },
            }

        for d in ('entry_dir', 'layout_dir', 'style_dir', 'output_dir'):
            if not os.path.exists(self.conf[d]):
                raise RuntimeError("%s %s does not exist" % (d, self.conf[d]))

        try:
            exec file(config) in self.conf
        except Exception, e:
            raise RuntimeError("Error running config: %s" % e)

        try: Entry.__bases__ += (self.conf['EntryMixin'],)
        except KeyError: pass

        method = lambda _self, obj, rep: magic(obj, rep, self.conf['charset'])
        Entry.magic = method

        # It would be nice if the factory could decide whether it needs to
        # open the file or not. All we need to know here is mtime, and all we
        # need for that is the filename/inode itself. But either way we need
        # to grab everything from the iterator up front so we can sort.

        self.box = mailbox.Maildir(self.conf['entry_dir'], Entry)
        self.entries = [e for e in self.box]
        self.entries.sort()

    def sing(self, entries=None, spath=None, dpath=None, what=None):
        """From the contents of spath, build output in dpath, based on the
        provided entries. For each entry in spath, will be called recursively
        with a tuple what representing the source and dest file. For any
        source files starting with __attr__ will recur several times based on
        which entries match each value of that attribute. For regularly named
        files, evaluate them as layout scripts if they are executable and
        simply copy them if they are not."""

        if not entries: entries = self.entries
        if not spath: spath = self.conf['layout_dir']
        if not dpath: dpath = self.conf['output_dir']

        def stale(dpath, spath, entries=None):
            """Test if the file named by dpath is nonexistent or older than
            either the file named by spath or any entry in the given list of
            entries. If --force has been turned on, always return True."""

            if self.force or not os.path.exists(dpath):
                return True
            else:
                dmtime = os.path.getmtime(dpath)
                smtimes = [os.path.getmtime(spath)]
                if entries: smtimes += [time.mktime(e.mtime) for e in entries]
                return dmtime < max(smtimes)

        if what:
            source, dest = what
            spath = os.path.join(spath, source)
            dpath = os.path.join(dpath, dest)
            if source not in self.conf['ignore']:
                if os.path.isfile(spath):
                    if os.stat(spath).st_mode & stat.S_IXUSR:
                        if stale(dpath, spath, entries):
                            self.sing_file(entries, spath, dpath)
                    else:
                        if stale(dpath, spath):
                            print 'Copied %s' % dpath
                            shutil.copyfile(spath, dpath)
                elif os.path.isdir(spath):
                    self.sing(entries, spath, dpath)
        else:
            if not os.path.isdir(dpath): os.makedirs(dpath)
            for f in os.listdir(spath):
                if f.startswith('__'):
                    self.sing_instances(entries, spath, dpath, f)
                else:
                    self.where.append(f)
                    self.sing(entries, spath, dpath, (f, f))
                    self.where.pop()

    def sing_instances(self, entries, spath, dpath, what):
        """Given a source and dest file in the tuple what, where the source
        starts with __attr__, group the provided entries by the values of that
        attribute over all the provided entries. For an entry e and attribute
        attr, e.attr may be an atomic value or a sequence of values. For each
        value so encountered, evaluate the source file given all entries in
        entries that match that value."""

        subst = what[:what.rindex('__')+2]

        def cheapiter(x):
            """DWIM-style iterator which, if given a sequence, will iterate
            over that sequence, unless it is a string type. For a string or
            any other atomic type, create an iterator which will return the
            given value once and then stop. This is a horrible kludge."""

            try:
                assert not isinstance((x), (str, unicode))
                for e in x: return iter(x)
                else: return iter(())
            except (AssertionError, TypeError):
                if x: return iter((x,))
                else: return iter(())

        inst = {}
        for e in entries:
            mv = getattr(e, subst[2:-2])
            for m in cheapiter(mv):
                inst.setdefault(repr(m), []).append(e)

        for k, entries in inst.iteritems():
            self.where.append(k)
            self.sing(entries, spath, dpath, (what, what.replace(subst, k)))
            self.where.pop()

    def expand(self, style, locals):
        """Open an EmPy file in the configuration's style directory, and
        evaluate it with the given locals."""
        style = os.path.join(self.conf['style_dir'], '%s.empy' % style)
        try:
            return em.expand(file(style).read(), locals)
        except Exception, e:
            raise RuntimeError("Error running style %s: %s" % (style, e))

    def sing_file(self, entries, spath, dpath):
        """Given an actual source and dest file, where the source is a layout
        script, evaluate it given the locals from config plus muse (ourself),
        write (a callback which actually writes the file), and entries (the
        given entries)."""

        def write(data):
            file(dpath, 'w').write(data)
            print 'Wrote %s' % dpath

        locals = self.conf['locals'].copy()
        locals['muse'] = self
        locals['write'] = write
        locals['entries'] = entries

        try:
            exec file(spath) in locals
        except Exception, e:
            raise RuntimeError("Error running layout %s: %s" % (spath, e))

class BaseEntry:
    """Base class for all entries. Initialized with an open file object, so it
    may be passed to maildir.Maildir as a factory class. Parses the file's
    contents as a Message object, setting a date attribute from the parsed
    date and an mtime attribute from the Maildir filename."""

    def __init__(self, fp):
        def fixdate(d):
            # For some bizarre reason, parsedate doesn't set wday/yday/isdst.
            return time.localtime(time.mktime(d))
        def getstamp(mpath):
            # djb says you're not supposed to do this. djb can bite me.
            stamp, id, host = os.path.split(mpath)[1].split('.')
            return int(stamp)

        self.msg = email.message_from_file(fp, Message)
        self.date = fixdate(email.Utils.parsedate(self.msg['Date']))
        self.mtime = time.localtime(getstamp(fp.name))

    def __cmp__(self, other):
        if other:
            return cmp(time.mktime(self.date), time.mktime(other.date))
        else:
            return 1

    rendered = {}
    def _prop_content(self):
        """Read in the message's body, strip any signature, and format using
        reStructedText."""

        try:
            return self.rendered[hash(self.msg)]
        except KeyError:
            s = self.msg.get_payload(decode=True)
            if not s: return ''

            try: s = s[:s.rindex('-- \n')]
            except ValueError: pass

            html = self.magic(publish_content(s), s[:100])
            return self.rendered.setdefault(hash(self.msg), html)

    byday = {}
    def _init_subject(self):
        """Get the contents of the Subject: header and a cleaned, uniq'd
        version of same."""

        try:
            subject = self.msg['Subject']
            cleaned = clean(subject, 3)
        except KeyError:
            subject = ''
            cleaned = 'entry'

        # Grab the namespace for the day of this entry
        day = self.byday.setdefault(self.date[0:3], UniqueDict())

        slug = day.setdefault(hash(self.msg), cleaned)
        return self.magic(subject, slug)

    def _init_id(self):
        """Get the Message-ID and a globally unique tag: URL based on it, for
        use in feeds."""

        try:
            id = self.msg['Message-Id'][1:-1]
            local, host = id.split('@')
            date = time.strftime('%Y-%m-%d', self.date)
            return self.magic(id, 'tag:%s,%s:%s' % (host, date, local))
        except KeyError:
            return ''

    def _init_author(self):
        """Get the real name portion of the From: address."""
        author, addr = email.Utils.parseaddr(self.msg.get('From'))
        return self.magic(author, clean(author))

    def _init_email(self):
        """Get the author's email address and a trivially spam-protected
        version of same."""
        try:
            author, addr = email.Utils.parseaddr(self.msg['From'])
            cleaned = addr.replace('@', ' at ')
            cleaned = cleaned.replace('.', ' dot ')
            cleaned = cleaned.replace('-', ' dash ')
            return self.magic(addr, cleaned)
        except KeyError:
            return ''

    def _init_tags(self):
        """Get a list of tags from the comma-delimited X-Tags: header."""
        try:
            tags = [t.strip() for t in self.msg['X-Tags'].split(',')]
            return [self.magic(t, clean(t)) for t in tags]
        except KeyError:
            return []

    def _init_year(self):
        """Get the year from the Date: header."""
        return self.magic(self.date[0], time.strftime('%Y', self.date))

    def _init_month(self):
        """Get the month from the Date: header."""
        return self.magic(self.date[1], time.strftime('%m', self.date))

    def _init_day(self):
        """Get the day of the month from the Date: header."""
        return self.magic(self.date[2], time.strftime('%d', self.date))

class Entry(BaseEntry):
    """Actual entry class. Will search the user-provided mixin class and then
    BaseEntry for methods of the form _init_*, and set the appropriate
    attribute on initialization, and also search for methods of the form
    _prop_* to provide properties to be evaluated on demand."""

    def __init__(self, fp):
        for _class in self.__class__.__bases__:
            try: _class.__init__(self, fp)
            except AttributeError: pass

            for k, v in _class.__dict__.iteritems():
                if k.startswith('_init_'):
                    setattr(self, k[6:], v(self))
                if k.startswith('_prop_'):
                    setattr(self.__class__, k[6:], property(v, None))

class Message(email.Message.Message):
    """Non-broken version of email's Message class. Returns unicode headers
    when necessary and raises KeyError when appropriate."""

    def __getitem__(self, item):
        header = email.Message.Message.__getitem__(self, item)
        if header:
            # worst. interface. ever.
            parts = email.Header.decode_header(header)
            decoded = []
            for p, enc in parts:
                if enc:
                    decoded.append(unicode(p, enc))
                else:
                    decoded.append(p)
            return ' '.join(decoded)
        else:
            raise KeyError

class UniqueDict(dict):
    """A read-only dict which munges its values so that they are unique. If an
    existing key has the value 'foo', attempting to set another key to 'foo'
    will cause it to become 'foo-1', then 'foo-2', etc. These numberings are
    stable as long as each key is assigned to in the same order; attempting to
    set an existing key will cause a ValueError. """

    def __getitem__(self, k):
        k, i = dict.__getitem__(self, k)
        if i: return '%s-%d' % (k, i)
        else: return k

    def __setitem__(self, k, v):
        if k in self: raise ValueError
        n = len([x for x, y in self.iteritems() if y[0] == v])
        dict.__setitem__(self, k, (v, n))

    # Yes, we must. Le sigh.
    def setdefault(self, key, failobj=None):
        if not self.has_key(key): self[key] = failobj
        return self[key]

def magic(obj, rep, enc):
    """Make obj into something suitable for passing to a layout. Returns an
    object exactly like obj, except its repr() is rep and both are encoded if
    they were unicode (layouts must serialize as a str, so giving them unicode
    data is not recommended). If you are not passing something that might be a
    unicode string, and do not care about using its repr() in your layout, you
    are not required to use this function."""

    if type(obj) is unicode: obj = obj.encode(enc)
    if type(rep) is unicode: rep = rep.encode(enc)

    _class = type("Magic", (type(obj),), {'__repr__': lambda self: rep})
    return _class(obj)

def clean(s, maxwords=None):
    """Split the given string into words, lowercase and strip all
    non-alphanumerics from them, and join them with '-'. If maxwords is given,
    limit the returned string to that many words. If the string is None,
    return None."""

    try:
        words = s.strip().lower().split()[:maxwords]
        words = [''.join([c for c in w if c.isalnum()]) for w in words]
        return '-'.join(words)
    except AttributeError:
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
@package  ion.services.sa.instrument.agent_status_builder
@author   Ian Katz
"""
from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient
from ooi.logging import log
from pyon.agent.agent import ResourceAgentClient
from pyon.core.bootstrap import IonObject
from pyon.core.exception import NotFound, Unauthorized, BadRequest

from interface.objects import ComputedValueAvailability, ComputedIntValue, ComputedDictValue, ComputedListValue
from interface.objects import AggregateStatusType, DeviceStatusType
from pyon.ion.resource import RT, PRED
from pyon.util.containers import DotDict

# possible ways of determining the type of a device driver
DriverTypingMethod = DotDict()
DriverTypingMethod.ByRR = 1
DriverTypingMethod.ByAgent = 2
DriverTypingMethod.ByException = 3

class AgentStatusBuilder(object):

    def __init__(self, process=None):
        """
        the process should be the "self" of a service instance
        """
        assert process
        self.process = process
        self.dtm = DriverTypingMethod.ByRR
        self.RR2 = None

        # make an internal pointer to this function so we can Mock it for testing
        self._get_agent_client = ResourceAgentClient

        if DriverTypingMethod.ByRR == self.dtm:
            self.RR2 = EnhancedResourceRegistryClient(process.clients.resource_registry)

    def set_status_computed_attributes(self, computed_attrs, values_dict=None, availability=None, reason=None):
        mappings = {AggregateStatusType.AGGREGATE_COMMS    : "communications_status_roll_up",
                    AggregateStatusType.AGGREGATE_POWER    : "power_status_roll_up",
                    AggregateStatusType.AGGREGATE_DATA     : "data_status_roll_up",
                    AggregateStatusType.AGGREGATE_LOCATION : "location_status_roll_up"}

        if values_dict is None:
            values_dict = {}

        for k, a in mappings.iteritems():
            if k in values_dict:
                status = ComputedIntValue(status=availability, value=values_dict[k], reason=reason)
            else:
                if None is reason: reason = "%s not in %s" % (k, values_dict)
                status = ComputedIntValue(status=ComputedValueAvailability.NOTAVAILABLE, reason=reason)
            setattr(computed_attrs, a, status)

    def set_status_computed_attributes_notavailable(self, computed_attrs, reason):
        self.set_status_computed_attributes(computed_attrs, None, ComputedValueAvailability.NOTAVAILABLE, reason)
        if hasattr(computed_attrs, "child_device_status"):
            computed_attrs.child_device_status = ComputedDictValue(status=ComputedValueAvailability.NOTAVAILABLE,
                                                                   reason=reason)

    def get_device_agent(self, device_id):
        if not device_id or device_id is None:
            return None, "No device ID was provided"

        try:
            h_agent = self._get_agent_client(device_id, process=self.process)
            log.debug("got the agent client here: %s for the device id: %s and process: %s",
                      h_agent, device_id, self.process)
        except NotFound:
            return None, "Could not connect to agent instance -- may not be running"

        except Unauthorized:
            return None, "The requester does not have the proper role to access the status of this agent"

        except AttributeError:
            return None, "Could not find an agent instance for this device id"

        return h_agent, ""

    # get a lookup table that includes child_agg_status + the parent device status as dev_id -> {AggStatusType: DeviceStatusType}
    def get_cumulative_status_dict(self, device_id):
        h_agent, reason = self.get_device_agent(device_id)
        log.trace("Got h_agent = %s, reason = %s", h_agent, reason)
        if None is h_agent:
            log.warn('no agent for device %s, reason=%s', device_id, reason)
            return None, reason

        # read child agg status
        try:
            #retrieve the platform status from the platform agent
            this_status = h_agent.get_agent(['aggstatus'])['aggstatus']
            log.debug("this_status for %s is %s", device_id, this_status)

        except Unauthorized:
            log.warn("The requester does not have the proper role to access the status of this agent")
            return None, "InstrumentDevice(get_agent) has been denied"

        out_status = {device_id: this_status}

        if DriverTypingMethod.ByAgent == self.dtm:
            # we're done if the agent doesn't support child_agg_status
            if not "child_agg_status" in [c.name for c in h_agent.get_capabilities()]:
                return out_status, None
        elif DriverTypingMethod.ByRR == self.dtm:
            device_obj = self.RR2.read(device_id)
            if RT.PlatformDevice != device_obj._get_type():
                return out_status, None

        try:
            child_agg_status = h_agent.get_agent(['child_agg_status'])['child_agg_status']
            log.debug('get_cumulative_status_dict child_agg_status : %s', child_agg_status)
            if child_agg_status:
                out_status.update(child_agg_status)
            return out_status, None
        except Unauthorized:
            log.warn("The requester does not have the proper role to access the child_agg_status of this agent")
            return out_status, "Error getting child status: 'child_agg_status' has been denied"

    #return this aggregate status, reason for fail, dict of device_id -> agg status
    def get_device_rollup_statuses_and_child_agg_status(self, device_id, child_device_ids=None, warn_missing=True):

        master_status_dict, reason = self.get_cumulative_status_dict(device_id)

        log.debug("Got master_status_dict = %s, reason = %s", master_status_dict, reason)
        if None is master_status_dict:
            return None, reason, None

        # no rolling up necessary for instruments (no child_device_ids list)
        if None is child_device_ids:
            return master_status_dict[device_id], None, None

        # if child device ids is not none, the developer had better be giving us a list
        assert isinstance(child_device_ids, list)

        log.debug("Computing rollup of devices%s in status from %s", child_device_ids + [device_id], master_status_dict)
        rollup_statuses = {}
        # 1. loop through the items in this device status,
        # 2. append all the statuses of that type from child devices,
        # 3. crush
        for stype, svalue in AggregateStatusType._str_map.iteritems():
            if not device_id in master_status_dict:
                log.warn("parent device %s not found in master_status_dict", device_id)
                this_status = {}
            else:
                this_status = master_status_dict[device_id]

            one_type_status_list = [this_status.get(stype, DeviceStatusType.STATUS_UNKNOWN)]
            for child_device_id in child_device_ids:
                if not child_device_id in master_status_dict:
                    if warn_missing:
                        log.warn("Child device '%s' of parent device '%s' not found in master_status_dict",
                                 child_device_id, device_id)
                else:
                    # get the dict of AggregateStatusType -> DeviceStatusType
                    child_statuses = master_status_dict[child_device_id]
                    one_type_status_list.append(child_statuses.get(stype, DeviceStatusType.STATUS_UNKNOWN))

            rollup_statuses[stype] = self._crush_status_list(one_type_status_list)

        log.debug("combined_status is %s", rollup_statuses)

        return rollup_statuses, None, master_status_dict

    # child_device_ids is None for instruments, a list for platforms
    def add_device_rollup_statuses_to_computed_attributes(self, device_id, extension_computed, child_device_ids=None):

        rollup_statuses, reason, child_agg_status = self.get_device_rollup_statuses_and_child_agg_status(device_id,
                                                                                child_device_ids)
        log.debug("Got rollup_statuses = %s, reason = %s", rollup_statuses, reason)

        if None is rollup_statuses:
            log.debug("setting status notavailable")
            self.set_status_computed_attributes_notavailable(extension_computed, reason)

            if hasattr(extension_computed, "child_device_status"):
                extension_computed.child_device_status = ComputedDictValue(status=ComputedValueAvailability.NOTAVAILABLE,
                                                                           reason=reason)
            return None

        # no rolling up necessary for instruments (no child_device_ids list)
        if None is child_device_ids:
            self.set_status_computed_attributes(extension_computed, rollup_statuses,
                                                ComputedValueAvailability.PROVIDED)
            return None

        # get child agg status if we can set child_device_status
        # todo: split into instrument and platform
        if child_agg_status and hasattr(extension_computed, "child_device_status"):
            crushed = dict([(k, self._crush_status_dict(v)) for k, v in child_agg_status.iteritems()])
            extension_computed.child_device_status = ComputedDictValue(status=ComputedValueAvailability.PROVIDED,
                                                                       value=crushed)

        self.set_status_computed_attributes(extension_computed, rollup_statuses,
                                            ComputedValueAvailability.PROVIDED)

        return child_agg_status

    def _crush_status_dict(self, values_dict):
        log.debug("crushing dict %s", values_dict)
        return self._crush_status_list(values_dict.values())

    def _crush_status_list(self, values_list):
        # reported status is worst (highest # value) of the component values
        status = max(values_list) if values_list else DeviceStatusType.STATUS_UNKNOWN
        log.debug("crushing list %s to value %s", values_list, status)
        return status

    #                                               keys becomes aggregate statuses in the same order
    def compute_status_list(self, child_agg_status, keys):
        ret = []
        if not isinstance(child_agg_status, dict):
            return ComputedListValue(reason="Top platform's child_agg_status is '%s'" % type(child_agg_status).__name__)

        for k in keys:
            # map None to UNKNOWN
            #if not type("") == type(k):
            #    raise BadRequest("attempted to compute_status_list with type(v) = %s : %s" % (type(k), k))
            if k in child_agg_status:
                ret.append(self._crush_status_dict(child_agg_status[k]))
            else:
                log.warn("Status for device '%s' not found in parent platform's child_agg_status", k)
                ret.append(DeviceStatusType.STATUS_UNKNOWN)

        return ComputedListValue(status=ComputedValueAvailability.PROVIDED,
                                 value=ret)

    def obtain_agent_calculation(self, device_id, result_container):
        ret = IonObject(result_container)

        h_agent, reason = self.get_device_agent(device_id)
        if None is h_agent:
            ret.status = ComputedValueAvailability.NOTAVAILABLE
            ret.reason = reason
        else:
            ret.status = ComputedValueAvailability.PROVIDED

        return h_agent, ret

    def _get_status_of_device(self, device_id):

        a_client, reason = self.get_device_agent(device_id)
        if None is a_client:
            return None, reason

        aggstatus = a_client.get_agent(['aggstatus'])['aggstatus']
        log.debug('get_aggregate_status_of_device status: %s', aggstatus)
        return aggstatus, ""

    def get_status_of_device(self, device_id):
        status, _ = self._get_status_of_device(device_id)
        if None is status:
            status = {}
        return status

    def get_aggregate_status_of_device(self, device_id):
        if  device_id is not None and type("") != type(device_id):
            errmsg = "get_aggregate_status_of_device passed bad type %s" % type(device_id)
            log.warn(errmsg)
            raise BadRequest(errmsg)

        aggstatus, reason = self._get_status_of_device(device_id)

        if None is aggstatus:
            DeviceStatusType.STATUS_UNKNOWN
        else:
            return self._crush_status_dict(aggstatus)

<commit_msg>add unauthorized exception catch for get_agent call in _get_status_of_device<commit_after>#!/usr/bin/env python

"""
@package  ion.services.sa.instrument.agent_status_builder
@author   Ian Katz
"""
from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient
from ooi.logging import log
from pyon.agent.agent import ResourceAgentClient
from pyon.core.bootstrap import IonObject
from pyon.core.exception import NotFound, Unauthorized, BadRequest

from interface.objects import ComputedValueAvailability, ComputedIntValue, ComputedDictValue, ComputedListValue
from interface.objects import AggregateStatusType, DeviceStatusType
from pyon.ion.resource import RT, PRED
from pyon.util.containers import DotDict

# possible ways of determining the type of a device driver
DriverTypingMethod = DotDict()
DriverTypingMethod.ByRR = 1
DriverTypingMethod.ByAgent = 2
DriverTypingMethod.ByException = 3

class AgentStatusBuilder(object):

    def __init__(self, process=None):
        """
        the process should be the "self" of a service instance
        """
        assert process
        self.process = process
        self.dtm = DriverTypingMethod.ByRR
        self.RR2 = None

        # make an internal pointer to this function so we can Mock it for testing
        self._get_agent_client = ResourceAgentClient

        if DriverTypingMethod.ByRR == self.dtm:
            self.RR2 = EnhancedResourceRegistryClient(process.clients.resource_registry)

    def set_status_computed_attributes(self, computed_attrs, values_dict=None, availability=None, reason=None):
        mappings = {AggregateStatusType.AGGREGATE_COMMS    : "communications_status_roll_up",
                    AggregateStatusType.AGGREGATE_POWER    : "power_status_roll_up",
                    AggregateStatusType.AGGREGATE_DATA     : "data_status_roll_up",
                    AggregateStatusType.AGGREGATE_LOCATION : "location_status_roll_up"}

        if values_dict is None:
            values_dict = {}

        for k, a in mappings.iteritems():
            if k in values_dict:
                status = ComputedIntValue(status=availability, value=values_dict[k], reason=reason)
            else:
                if None is reason: reason = "%s not in %s" % (k, values_dict)
                status = ComputedIntValue(status=ComputedValueAvailability.NOTAVAILABLE, reason=reason)
            setattr(computed_attrs, a, status)

    def set_status_computed_attributes_notavailable(self, computed_attrs, reason):
        self.set_status_computed_attributes(computed_attrs, None, ComputedValueAvailability.NOTAVAILABLE, reason)
        if hasattr(computed_attrs, "child_device_status"):
            computed_attrs.child_device_status = ComputedDictValue(status=ComputedValueAvailability.NOTAVAILABLE,
                                                                   reason=reason)

    def get_device_agent(self, device_id):
        if not device_id or device_id is None:
            return None, "No device ID was provided"

        try:
            h_agent = self._get_agent_client(device_id, process=self.process)
            log.debug("got the agent client here: %s for the device id: %s and process: %s",
                      h_agent, device_id, self.process)
        except NotFound:
            return None, "Could not connect to agent instance -- may not be running"

        except Unauthorized:
            return None, "The requester does not have the proper role to access the status of this agent"

        except AttributeError:
            return None, "Could not find an agent instance for this device id"

        return h_agent, ""

    # get a lookup table that includes child_agg_status + the parent device status as dev_id -> {AggStatusType: DeviceStatusType}
    def get_cumulative_status_dict(self, device_id):
        h_agent, reason = self.get_device_agent(device_id)
        log.trace("Got h_agent = %s, reason = %s", h_agent, reason)
        if None is h_agent:
            log.warn('no agent for device %s, reason=%s', device_id, reason)
            return None, reason

        # read child agg status
        try:
            #retrieve the platform status from the platform agent
            this_status = h_agent.get_agent(['aggstatus'])['aggstatus']
            log.debug("this_status for %s is %s", device_id, this_status)

        except Unauthorized:
            log.warn("The requester does not have the proper role to access the status of this agent")
            return None, "InstrumentDevice(get_agent) has been denied"

        out_status = {device_id: this_status}

        if DriverTypingMethod.ByAgent == self.dtm:
            # we're done if the agent doesn't support child_agg_status
            if not "child_agg_status" in [c.name for c in h_agent.get_capabilities()]:
                return out_status, None
        elif DriverTypingMethod.ByRR == self.dtm:
            device_obj = self.RR2.read(device_id)
            if RT.PlatformDevice != device_obj._get_type():
                return out_status, None

        try:
            child_agg_status = h_agent.get_agent(['child_agg_status'])['child_agg_status']
            log.debug('get_cumulative_status_dict child_agg_status : %s', child_agg_status)
            if child_agg_status:
                out_status.update(child_agg_status)
            return out_status, None
        except Unauthorized:
            log.warn("The requester does not have the proper role to access the child_agg_status of this agent")
            return out_status, "Error getting child status: 'child_agg_status' has been denied"

    #return this aggregate status, reason for fail, dict of device_id -> agg status
    def get_device_rollup_statuses_and_child_agg_status(self, device_id, child_device_ids=None, warn_missing=True):

        master_status_dict, reason = self.get_cumulative_status_dict(device_id)

        log.debug("Got master_status_dict = %s, reason = %s", master_status_dict, reason)
        if None is master_status_dict:
            return None, reason, None

        # no rolling up necessary for instruments (no child_device_ids list)
        if None is child_device_ids:
            return master_status_dict[device_id], None, None

        # if child device ids is not none, the developer had better be giving us a list
        assert isinstance(child_device_ids, list)

        log.debug("Computing rollup of devices%s in status from %s", child_device_ids + [device_id], master_status_dict)
        rollup_statuses = {}
        # 1. loop through the items in this device status,
        # 2. append all the statuses of that type from child devices,
        # 3. crush
        for stype, svalue in AggregateStatusType._str_map.iteritems():
            if not device_id in master_status_dict:
                log.warn("parent device %s not found in master_status_dict", device_id)
                this_status = {}
            else:
                this_status = master_status_dict[device_id]

            one_type_status_list = [this_status.get(stype, DeviceStatusType.STATUS_UNKNOWN)]
            for child_device_id in child_device_ids:
                if not child_device_id in master_status_dict:
                    if warn_missing:
                        log.warn("Child device '%s' of parent device '%s' not found in master_status_dict",
                                 child_device_id, device_id)
                else:
                    # get the dict of AggregateStatusType -> DeviceStatusType
                    child_statuses = master_status_dict[child_device_id]
                    one_type_status_list.append(child_statuses.get(stype, DeviceStatusType.STATUS_UNKNOWN))

            rollup_statuses[stype] = self._crush_status_list(one_type_status_list)

        log.debug("combined_status is %s", rollup_statuses)

        return rollup_statuses, None, master_status_dict

    # child_device_ids is None for instruments, a list for platforms
    def add_device_rollup_statuses_to_computed_attributes(self, device_id, extension_computed, child_device_ids=None):

        rollup_statuses, reason, child_agg_status = self.get_device_rollup_statuses_and_child_agg_status(device_id,
                                                                                child_device_ids)
        log.debug("Got rollup_statuses = %s, reason = %s", rollup_statuses, reason)

        if None is rollup_statuses:
            log.debug("setting status notavailable")
            self.set_status_computed_attributes_notavailable(extension_computed, reason)

            if hasattr(extension_computed, "child_device_status"):
                extension_computed.child_device_status = ComputedDictValue(status=ComputedValueAvailability.NOTAVAILABLE,
                                                                           reason=reason)
            return None

        # no rolling up necessary for instruments (no child_device_ids list)
        if None is child_device_ids:
            self.set_status_computed_attributes(extension_computed, rollup_statuses,
                                                ComputedValueAvailability.PROVIDED)
            return None

        # get child agg status if we can set child_device_status
        # todo: split into instrument and platform
        if child_agg_status and hasattr(extension_computed, "child_device_status"):
            crushed = dict([(k, self._crush_status_dict(v)) for k, v in child_agg_status.iteritems()])
            extension_computed.child_device_status = ComputedDictValue(status=ComputedValueAvailability.PROVIDED,
                                                                       value=crushed)

        self.set_status_computed_attributes(extension_computed, rollup_statuses,
                                            ComputedValueAvailability.PROVIDED)

        return child_agg_status

    def _crush_status_dict(self, values_dict):
        log.debug("crushing dict %s", values_dict)
        return self._crush_status_list(values_dict.values())

    def _crush_status_list(self, values_list):
        # reported status is worst (highest # value) of the component values
        status = max(values_list) if values_list else DeviceStatusType.STATUS_UNKNOWN
        log.debug("crushing list %s to value %s", values_list, status)
        return status

    #                                               keys becomes aggregate statuses in the same order
    def compute_status_list(self, child_agg_status, keys):
        ret = []
        if not isinstance(child_agg_status, dict):
            return ComputedListValue(reason="Top platform's child_agg_status is '%s'" % type(child_agg_status).__name__)

        for k in keys:
            # map None to UNKNOWN
            #if not type("") == type(k):
            #    raise BadRequest("attempted to compute_status_list with type(v) = %s : %s" % (type(k), k))
            if k in child_agg_status:
                ret.append(self._crush_status_dict(child_agg_status[k]))
            else:
                log.warn("Status for device '%s' not found in parent platform's child_agg_status", k)
                ret.append(DeviceStatusType.STATUS_UNKNOWN)

        return ComputedListValue(status=ComputedValueAvailability.PROVIDED,
                                 value=ret)

    def obtain_agent_calculation(self, device_id, result_container):
        ret = IonObject(result_container)

        h_agent, reason = self.get_device_agent(device_id)
        if None is h_agent:
            ret.status = ComputedValueAvailability.NOTAVAILABLE
            ret.reason = reason
        else:
            ret.status = ComputedValueAvailability.PROVIDED

        return h_agent, ret

    def _get_status_of_device(self, device_id):

        a_client, reason = self.get_device_agent(device_id)
        if None is a_client:
            return None, reason
        try:
            aggstatus = a_client.get_agent(['aggstatus'])['aggstatus']
            log.debug('get_aggregate_status_of_device status: %s', aggstatus)
            return aggstatus, ""
        except Unauthorized:
            log.warn("The requester does not have the proper role to access the status of this agent")
            return None, "InstrumentDevice(get_agent) has been denied"

    def get_status_of_device(self, device_id):
        status, _ = self._get_status_of_device(device_id)
        if None is status:
            status = {}
        return status

    def get_aggregate_status_of_device(self, device_id):
        if  device_id is not None and type("") != type(device_id):
            errmsg = "get_aggregate_status_of_device passed bad type %s" % type(device_id)
            log.warn(errmsg)
            raise BadRequest(errmsg)

        aggstatus, reason = self._get_status_of_device(device_id)

        if None is aggstatus:
            DeviceStatusType.STATUS_UNKNOWN
        else:
            return self._crush_status_dict(aggstatus)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Presence analyzer unit tests."""
import os.path
import json
import datetime
import unittest

from presence_analyzer import (
    main,
    views,
    utils,
    )

TEST_DATA_CSV = os.path.join(
    os.path.dirname(__file__), '..', '..', 'runtime', 'data', 'test_data.csv'
)

TEST_DATA_XML = os.path.join(
    os.path.dirname(__file__), '..', '..', 'runtime', 'data', 'test_data.xml'
)

TEST_CASHED_DATA = os.path.join(
    os.path.dirname(__file__), '..', '..',
    'runtime', 'data', 'test_cache_data.csv'
)

# pylint: disable=E1103
class PresenceAnalyzerViewsTestCase(unittest.TestCase):
    """Views tests."""

    def setUp(self):
        """Before each test, set up a environment."""
        main.app.config.update({
            'DATA_CSV': TEST_DATA_CSV,
            'DATA_XML': TEST_DATA_XML,
            })
        self.client = main.app.test_client()

    def tearDown(self):
        """Get rid of unused objects after each test."""
        pass

    def test_mainpage(self):
        """Test main page redirect."""
        resp = self.client.get('/')
        self.assertEqual(resp.status_code, 302)
        assert resp.headers['Location'].endswith('/presence_weekday.html')

    def test_api_users(self):
        """Test users listing."""
        resp = self.client.get('/api/v1/users')
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.content_type, 'application/json')
        data = json.loads(resp.data)
        self.assertEqual(len(data), 2)
        self.assertListEqual(data, [
            {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/151',
                u'name': u'Not F.',
                u'user_id': u'11'
            }, {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/165',
                u'name': u'Rando M.',
                u'user_id': u'10'
            }
        ])

    def test_users_view(self):
        """Test user view"""
        result = self.client.get('/api/v1/mean_time_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertIsInstance(data[0], list)
        self.assertListEqual(data, [
            [u'Mon', 24123],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 22984.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ]
            )

    def test_mean_time_weekday_view(self):
        """Test view of mean time for user"""
        result = self.client.get('/api/v1/mean_time_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertIsInstance(data[0], list)
        self.assertListEqual(data, [
            [u'Mon', 24123.0],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 22984.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ],
            )

    def test_presence_weekday_view(self):
        """Test view of weekday time for user"""
        result = self.client.get('/api/v1/presence_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Weekday', u'Presence (s)'],
            [u'Mon', 24123.0],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 45968.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ],
            )

    def test_presence_start_end_view(self):
        """Test view of start-end time for user"""
        result = self.client.get('/api/v1/presence_start_end/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Mon', 33134.0, 57257.0],
            [u'Tue', 33590.0, 50154.0],
            [u'Wed', 33206.0, 58527.0],
            [u'Thu', 35602.0, 58586.0],
            [u'Fri', 47816.0, 54242.0],
            [u'Sat', 0, 0],
            [u'Sun', 0, 0]
            ]
            )

        result = self.client.get('/api/v1/presence_start_end/10')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Mon', 0, 0],
            [u'Tue', 34745.0, 64792.0],
            [u'Wed', 33592.0, 58057.0],
            [u'Thu', 38926.0, 62631.0],
            [u'Fri', 0, 0],
            [u'Sat', 0, 0],
            [u'Sun', 0, 0]
            ]
            )

    def test_template_render(self):
        """Test rendering templates"""
        data_list = [
            {
                'url': "/presence_weekday.html",
                "unique": "Presence by weekday"
            }, {
                'url': "/mean_time_weekday.html",
                "unique": "Presence mean time by weekday"
            }, {
                'url': "/presence_start_end.html",
                "unique": "Presence start-end weekday"
            }
        ]
        for data in data_list:
            result = self.client.get(data['url'])
            self.assertIsNotNone(result)
            self.assertEqual(result.status_code, 200)
            self.assertEqual(result.content_type, 'text/html; charset=utf-8')
            self.assertIn(data['unique'], result.data)

    def test_template_render_secon(self):
        """Second testing function simulating real tests
        Especially for you
        """
        wrong_data = [
            {'url': "/basic.html", "code": 404},
            {'url': "/not_existing.html", "code": 404}
        ]
        for data in wrong_data:
            result = self.client.get(data['url'])
            self.assertEqual(result.status_code, data['code'])

class PresenceAnalyzerUtilsTestCase(unittest.TestCase):
    """Utility functions tests."""

    def setUp(self):
        """Before each test, set up a environment."""
        main.app.config.update({
            'DATA_CSV': TEST_DATA_CSV,
            'DATA_XML': TEST_DATA_XML,
        })
        utils.cache_data = {}

    def tearDown(self):
        """Get rid of unused objects after each test."""
        pass

    def test_get_data(self):
        """Test parsing of CSV file."""
        data = utils.get_data()
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

    def test_group_by_weekday(self):
        """Test groups entries by weekday"""
        data = utils.get_data()
        result = utils.group_by_weekday(data[10])
        self.assertEqual(len(result), 7)
        self.assertIsInstance(result, dict)
        self.assertIsInstance(result[2], list)
        self.assertDictEqual(result, {
            0: [],
            1: [utils.interval
                (data[10][datetime.date(2013, 9, 10)]['start'],
                 data[10][datetime.date(2013, 9, 10)]['end']
                 )
                ],
            2: [utils.interval
                (data[10][datetime.date(2013, 9, 11)]['start'],
                 data[10][datetime.date(2013, 9, 11)]['end']
                 )
                ],
            3: [utils.interval
                (data[10][datetime.date(2013, 9, 12)]['start'],
                 data[10][datetime.date(2013, 9, 12)]['end']
                 )
                ],
            4: [],
            5: [],
            6: [],
            }
            )

        self.assertEqual(len(result[3]), 1)

    def test_group_by_start_end(self):
        """Test group by start-end"""
        data = utils.get_data()
        self.assertIsNotNone(data)
        result = utils.group_by_start_end(data[11])
        self.assertIsNotNone(result)
        self.assertIsInstance(data, dict)
        self.assertEqual(len(result), 7)
        self.assertDictEqual(result, {
            0: {'start': [33134], 'end': [57257]},
            1: {'start': [33590], 'end': [50154]},
            2: {'start': [33206], 'end': [58527]},
            3: {'start': [37116, 34088], 'end': [60085, 57087]},
            4: {'start': [47816], 'end': [54242]},
            5: {'start': [], 'end': []},
            6: {'start': [], 'end': []}
            }
            )

        result = utils.group_by_start_end(data[10])
        self.assertIsNotNone(result)
        self.assertIsInstance(data, dict)
        self.assertEqual(len(result), 7)
        self.assertDictEqual(result, {
            0: {'start': [], 'end': []},
            1: {'start': [34745], 'end': [64792]},
            2: {'start': [33592], 'end': [58057]},
            3: {'start': [38926], 'end': [62631]},
            4: {'start': [], 'end': []},
            5: {'start': [], 'end': []},
            6: {'start': [], 'end': []}
            }
            )

    def test_seconds_since_midnight(self):
        """Test calculated amount of seconds since midnight"""
        sample_date = datetime.time(10, 10, 10)
        self.assertEqual(utils.seconds_since_midnight(sample_date),
                         10*3600 + 10*60 + 10)

    def test_interval(self):
        """Test calculates inverval in seconds between two
        datetime.time objects.
        """
        sample_date_1 = datetime.time(10, 10, 10)
        sample_date_2 = datetime.time(12, 12, 12)
        self.assertEqual(utils.interval(sample_date_1, sample_date_2),
                         2*3600 + 2*60 + 2)

    def test_mean(self):
        """Test calculates arithmetic mean."""
        self.assertIsNotNone(utils.mean([]))
        self.assertIsInstance(utils.mean([1, 1.4, 5, 1.3]), float)
        self.assertEqual(utils.mean([1, 2, 3, 4]), 2.5)
        self.assertEqual(utils.mean([-1, -2, 3, 4]), 1)
        self.assertEqual(utils.mean([1.1, 1.2, 1.3, 1.4]), 1.25)
        self.assertEqual(utils.mean([]), 0)

    def test_get_users_data(self):
        """Test returned data from xml"""
        data = utils.get_users_data()
        self.assertDictEqual(data, {
            '10': {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/165',
                u'name': 'Rando M.',
            },
            '11': {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/151',
                u'name': 'Not F.',
            }
        })

    def test_cache(self):
        """ Test cache decorator"""
        data = [1, 2, 3, 4, 5]

        @utils.cache("decorated function", 600)
        def decorated_function():
            return data

        for i in xrange(5):
            result = decorated_function()
            self.assertEqual(result, data)
            utils.get_data()
        self.assertIn("decorated function", utils.cache_data)

        utils.get_data()
        data = utils.cache_data['get_data']['result']
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

        main.app.config.update({'DATA_CSV': TEST_CASHED_DATA})

        data = utils.get_data()
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

        utils.cache_data = {}
        data = utils.get_data()
        self.assertEqual(data, {
            10: {
                datetime.date(2013, 9, 10): {
                    'end': datetime.time(17, 59, 52),
                    'start': datetime.time(9, 39, 5)
                }
            }
        })

def suite():
    """Default test suite."""
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(PresenceAnalyzerViewsTestCase))
    suite.addTest(unittest.makeSuite(PresenceAnalyzerUtilsTestCase))
    return suite

if __name__ == '__main__':
    unittest.main()
<commit_msg>[2533] Corrected spelling.<commit_after># -*- coding: utf-8 -*-
"""Presence analyzer unit tests."""
import os.path
import json
import datetime
import unittest

from presence_analyzer import (
    main,
    views,
    utils,
    )

TEST_DATA_CSV = os.path.join(
    os.path.dirname(__file__), '..', '..', 'runtime', 'data', 'test_data.csv'
)

TEST_DATA_XML = os.path.join(
    os.path.dirname(__file__), '..', '..', 'runtime', 'data', 'test_data.xml'
)

TEST_CACHED_DATA = os.path.join(
    os.path.dirname(__file__), '..', '..',
    'runtime', 'data', 'test_cache_data.csv'
)

# pylint: disable=E1103
class PresenceAnalyzerViewsTestCase(unittest.TestCase):
    """Views tests."""

    def setUp(self):
        """Before each test, set up a environment."""
        main.app.config.update({
            'DATA_CSV': TEST_DATA_CSV,
            'DATA_XML': TEST_DATA_XML,
            })
        self.client = main.app.test_client()

    def tearDown(self):
        """Get rid of unused objects after each test."""
        pass

    def test_mainpage(self):
        """Test main page redirect."""
        resp = self.client.get('/')
        self.assertEqual(resp.status_code, 302)
        assert resp.headers['Location'].endswith('/presence_weekday.html')

    def test_api_users(self):
        """Test users listing."""
        resp = self.client.get('/api/v1/users')
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.content_type, 'application/json')
        data = json.loads(resp.data)
        self.assertEqual(len(data), 2)
        self.assertListEqual(data, [
            {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/151',
                u'name': u'Not F.',
                u'user_id': u'11'
            }, {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/165',
                u'name': u'Rando M.',
                u'user_id': u'10'
            }
        ])

    def test_users_view(self):
        """Test user view"""
        result = self.client.get('/api/v1/mean_time_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertIsInstance(data[0], list)
        self.assertListEqual(data, [
            [u'Mon', 24123],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 22984.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ]
            )

    def test_mean_time_weekday_view(self):
        """Test view of mean time for user"""
        result = self.client.get('/api/v1/mean_time_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertIsInstance(data[0], list)
        self.assertListEqual(data, [
            [u'Mon', 24123.0],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 22984.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ],
            )

    def test_presence_weekday_view(self):
        """Test view of weekday time for user"""
        result = self.client.get('/api/v1/presence_weekday/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Weekday', u'Presence (s)'],
            [u'Mon', 24123.0],
            [u'Tue', 16564.0],
            [u'Wed', 25321.0],
            [u'Thu', 45968.0],
            [u'Fri', 6426.0],
            [u'Sat', 0],
            [u'Sun', 0]
            ],
            )

    def test_presence_start_end_view(self):
        """Test view of start-end time for user"""
        result = self.client.get('/api/v1/presence_start_end/11')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Mon', 33134.0, 57257.0],
            [u'Tue', 33590.0, 50154.0],
            [u'Wed', 33206.0, 58527.0],
            [u'Thu', 35602.0, 58586.0],
            [u'Fri', 47816.0, 54242.0],
            [u'Sat', 0, 0],
            [u'Sun', 0, 0]
            ]
            )

        result = self.client.get('/api/v1/presence_start_end/10')
        self.assertIsNotNone(result)
        self.assertEqual(result.status_code, 200)
        self.assertEqual(result.content_type, 'application/json')
        data = json.loads(result.data)
        self.assertListEqual(data, [
            [u'Mon', 0, 0],
            [u'Tue', 34745.0, 64792.0],
            [u'Wed', 33592.0, 58057.0],
            [u'Thu', 38926.0, 62631.0],
            [u'Fri', 0, 0],
            [u'Sat', 0, 0],
            [u'Sun', 0, 0]
            ]
            )

    def test_template_render(self):
        """Test rendering templates"""
        data_list = [
            {
                'url': "/presence_weekday.html",
                "unique": "Presence by weekday"
            }, {
                'url': "/mean_time_weekday.html",
                "unique": "Presence mean time by weekday"
            }, {
                'url': "/presence_start_end.html",
                "unique": "Presence start-end weekday"
            }
        ]
        for data in data_list:
            result = self.client.get(data['url'])
            self.assertIsNotNone(result)
            self.assertEqual(result.status_code, 200)
            self.assertEqual(result.content_type, 'text/html; charset=utf-8')
            self.assertIn(data['unique'], result.data)

    def test_template_render_secon(self):
        """Second testing function simulating real tests
        Especially for you
        """
        wrong_data = [
            {'url': "/basic.html", "code": 404},
            {'url': "/not_existing.html", "code": 404}
        ]
        for data in wrong_data:
            result = self.client.get(data['url'])
            self.assertEqual(result.status_code, data['code'])

class PresenceAnalyzerUtilsTestCase(unittest.TestCase):
    """Utility functions tests."""

    def setUp(self):
        """Before each test, set up a environment."""
        main.app.config.update({
            'DATA_CSV': TEST_DATA_CSV,
            'DATA_XML': TEST_DATA_XML,
        })
        utils.cache_data = {}

    def tearDown(self):
        """Get rid of unused objects after each test."""
        pass

    def test_get_data(self):
        """Test parsing of CSV file."""
        data = utils.get_data()
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

    def test_group_by_weekday(self):
        """Test groups entries by weekday"""
        data = utils.get_data()
        result = utils.group_by_weekday(data[10])
        self.assertEqual(len(result), 7)
        self.assertIsInstance(result, dict)
        self.assertIsInstance(result[2], list)
        self.assertDictEqual(result, {
            0: [],
            1: [utils.interval
                (data[10][datetime.date(2013, 9, 10)]['start'],
                 data[10][datetime.date(2013, 9, 10)]['end']
                 )
                ],
            2: [utils.interval
                (data[10][datetime.date(2013, 9, 11)]['start'],
                 data[10][datetime.date(2013, 9, 11)]['end']
                 )
                ],
            3: [utils.interval
                (data[10][datetime.date(2013, 9, 12)]['start'],
                 data[10][datetime.date(2013, 9, 12)]['end']
                 )
                ],
            4: [],
            5: [],
            6: [],
            }
            )

        self.assertEqual(len(result[3]), 1)

    def test_group_by_start_end(self):
        """Test group by start-end"""
        data = utils.get_data()
        self.assertIsNotNone(data)
        result = utils.group_by_start_end(data[11])
        self.assertIsNotNone(result)
        self.assertIsInstance(data, dict)
        self.assertEqual(len(result), 7)
        self.assertDictEqual(result, {
            0: {'start': [33134], 'end': [57257]},
            1: {'start': [33590], 'end': [50154]},
            2: {'start': [33206], 'end': [58527]},
            3: {'start': [37116, 34088], 'end': [60085, 57087]},
            4: {'start': [47816], 'end': [54242]},
            5: {'start': [], 'end': []},
            6: {'start': [], 'end': []}
            }
            )

        result = utils.group_by_start_end(data[10])
        self.assertIsNotNone(result)
        self.assertIsInstance(data, dict)
        self.assertEqual(len(result), 7)
        self.assertDictEqual(result, {
            0: {'start': [], 'end': []},
            1: {'start': [34745], 'end': [64792]},
            2: {'start': [33592], 'end': [58057]},
            3: {'start': [38926], 'end': [62631]},
            4: {'start': [], 'end': []},
            5: {'start': [], 'end': []},
            6: {'start': [], 'end': []}
            }
            )

    def test_seconds_since_midnight(self):
        """Test calculated amount of seconds since midnight"""
        sample_date = datetime.time(10, 10, 10)
        self.assertEqual(utils.seconds_since_midnight(sample_date),
                         10*3600 + 10*60 + 10)

    def test_interval(self):
        """Test calculates inverval in seconds between two
        datetime.time objects.
        """
        sample_date_1 = datetime.time(10, 10, 10)
        sample_date_2 = datetime.time(12, 12, 12)
        self.assertEqual(utils.interval(sample_date_1, sample_date_2),
                         2*3600 + 2*60 + 2)

    def test_mean(self):
        """Test calculates arithmetic mean."""
        self.assertIsNotNone(utils.mean([]))
        self.assertIsInstance(utils.mean([1, 1.4, 5, 1.3]), float)
        self.assertEqual(utils.mean([1, 2, 3, 4]), 2.5)
        self.assertEqual(utils.mean([-1, -2, 3, 4]), 1)
        self.assertEqual(utils.mean([1.1, 1.2, 1.3, 1.4]), 1.25)
        self.assertEqual(utils.mean([]), 0)

    def test_get_users_data(self):
        """Test returned data from xml"""
        data = utils.get_users_data()
        self.assertDictEqual(data, {
            '10': {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/165',
                u'name': 'Rando M.',
            },
            '11': {
                u'avatar': u'https://intranet.stxnext.pl/api/images/users/151',
                u'name': 'Not F.',
            }
        })

    def test_cache(self):
        """ Test cache decorator"""
        data = [1, 2, 3, 4, 5]

        @utils.cache("decorated function", 600)
        def decorated_function():
            return data

        for i in xrange(5):
            result = decorated_function()
            self.assertEqual(result, data)
            utils.get_data()
        self.assertIn("decorated function", utils.cache_data)

        utils.get_data()
        data = utils.cache_data['get_data']['result']
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

        main.app.config.update({'DATA_CSV': TEST_CACHED_DATA})

        data = utils.get_data()
        self.assertIsInstance(data, dict)
        self.assertItemsEqual(data.keys(), [10, 11])
        sample_date = datetime.date(2013, 9, 10)
        self.assertIn(sample_date, data[10])
        self.assertItemsEqual(data[10][sample_date].keys(), ['start', 'end'])
        self.assertEqual(data[10][sample_date]['start'],
                         datetime.time(9, 39, 5))

        utils.cache_data = {}
        data = utils.get_data()
        self.assertEqual(data, {
            10: {
                datetime.date(2013, 9, 10): {
                    'end': datetime.time(17, 59, 52),
                    'start': datetime.time(9, 39, 5)
                }
            }
        })

def suite():
    """Default test suite."""
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(PresenceAnalyzerViewsTestCase))
    suite.addTest(unittest.makeSuite(PresenceAnalyzerUtilsTestCase))
    return suite

if __name__ == '__main__':
    unittest.main()
<|endoftext|>