<commit_before>'''loadconfig python library'''
from __future__ import print_function
__all__ = ['Config', 'Odict', '__version__']

__author__ = 'Daniel Mizyrycki'
__version__ = '0.1'

from os.path import dirname, abspath
import sys
sys.path.insert(0, dirname(abspath(__file__)))

from collections import OrderedDict
from copy import deepcopy
from itertools import count
from lib import (delregex, dfl, findregex, flatten, read_config_file,
    read_file, _clg_parse, _get_option)
from os import environ
from py6 import cStringIO, shlex_quote
from string import Template
import yaml
from yaml.scanner import ScannerError

class Odict(OrderedDict):
    r'''Add more readable representation to OrderedDict using yaml.

    >>> d = Odict('{a: 1, b: {c: 3}}')
    >>> repr(d)
    '{a: 1, b: {c: 3}}'
    >>> str(d)
    'a: 1\nb:\n  c: 3'
    >>> d == Odict(d)
    True
    >>> d.b.c
    3
    >>> d._p
    a: 1
    b:
      c: 3
    '''
    def __init__(self, *args, **kwargs):
        '''Process first argument as yaml if it is a string'''
        args = self._process_args(*args, **kwargs)
        super(Odict, self).__init__(*args, **kwargs)

    def _process_args(self, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], str):
            try:
                pair_list = self.load(args[0])
            except ScannerError:
                # Yaml needs {} on multi-key single-line strings
                pair_list = self.load('{{{}}}'.format(args[0]))
            args = [pair_list] if pair_list else []
        return args

    def update(self, *args, **kwargs):
        '''Add yaml string as posible argument

        >>> d = Odict('hi: there')
        >>> d.hi
        'there'
        >>> d.update({'hi': 'Liss'})
        >>> d.hi
        'Liss'
        '''
        args = self._process_args(*args, **kwargs)
        super(Odict, self).update(*args, **kwargs)
        # Remove '_' key possible used by include
        if '_' in self:
            del self['_']

    def __getattr__(self, name):
        '''Access Odict key as attribute

        >>> c = Config('activity: [hike, bike, scuba dive, run]')
        >>> c.activity
        ['hike', 'bike', 'scuba dive', 'run']
        '''
        if name in self:
            return self[name]
        if name in ['_OrderedDict__root', '__deepcopy__']:
            return super(Odict, self).__getattr__(name)

    def __delattr__(self, name):
        '''Delete Odict key from attribute
        >>> c = Config('activity: [hike, bike, scuba dive, run]')
        >>> del c.activity
        >>> c
        {}
        '''
        del self[name]

    def __setattr__(self, name, value):
        '''Set Odict key from attribute

        >>> c = Config('activity: [hike, scuba dive]')
        >>> c.place = 'Hawaii'
        >>> c
        {activity: [hike, scuba dive], place: Hawaii}
        '''
        if name in ['_OrderedDict__root', '_OrderedDict__hardroot',
         '_OrderedDict__map']:
            return super(Odict, self).__setattr__(name, value)
        self[name] = value

    def __str__(self):
        return Odict.dump(self, False)

    def __repr__(self):
        return Odict.dump(self)

    # Convenient shortcuts
    _r = property(__repr__)
    _s = property(__str__)
    _p = property(lambda x: print(str(x)))

    @staticmethod
    def load(yaml_string):
        '''Return pair list from a yaml string'''
        class Loader(yaml.SafeLoader):
            yaml_mapping = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
            _data = ''

            def __init__(self, *args, **kwargs):
                super(Loader, self).__init__(*args, **kwargs)
                self.add_constructor(self.yaml_mapping, self.construct_mapping)
                self.add_constructor('!env', self.env)
                self.add_constructor('!read', self.read)
                self.add_constructor('!include', self.include)
                self.add_constructor('!expand', self.expand)

            def env(self, safeloader, node):
                node = self.construct_scalar(node)
                if node.upper() in environ:
                    return {node: environ[node.upper()]}
                return {node: ''}

            def read(self, safeloader, node):
                node = self.construct_scalar(node)
                return read_file(node)

            def include(self, safeloader, node):
                node = self.construct_scalar(node)
                filepath, sep, key = node.partition(':')
                self._data = yaml.load(read_file(filepath), Loader)
                return self.subkey(key)

            def expand(self, safeloader, node):
                key = self.construct_scalar(node)
                self._data = safeloader._data
                return self.subkey(key)

            def subkey(self, key):
                if not self._data:
                    return ''
                data = deepcopy(self._data)
                if key == '&':
                    return ''
                for k in key.split(':'):
                    if not k:
                        return data
                    data = data.get(k)
                    if data is None:
                        return ''
                return data

            def construct_mapping(self, safeloader, node):
                safeloader.flatten_mapping(node)
                return Odict(safeloader.construct_pairs(node))

        return yaml.load(yaml_string, Loader)

    @staticmethod
    def dump(self, default_flow_style=True):
        '''Serialize odict into yaml string'''
        class Dumper(yaml.SafeDumper):
            def __init__(self, *args, **kwargs):
                super(Dumper, self).__init__(*args, **kwargs)
                self.ignore_aliases = lambda self: True
                self.add_representer(Odict, lambda self, data:
                    self.represent_dict(data.items()))

            def increase_indent(self, flow=False, indentless=False):
                return super(Dumper, self).increase_indent(flow, False)

        stream = cStringIO()
        yaml.dump(self, stream, Dumper, default_flow_style=default_flow_style)
        return stream.getvalue()[:-1]

class Config(Odict):
    '''Config class for programs.

    The Config class is designed to be a versatile interface to load a config
    object for use in python and shell programs.
    Source data comes from config_data object and from interpreting the args
    parameter (usually sys.args) from command line.
    config_data object can be a yaml string or a dictionary (eg: 1 doctest).
    args list take any number of arguments. Optionally some arguments can be
    in the form of -C (for config file) and -E (for extra) options (2).
    The python clg package is used to parse normal cli arguments using the
    special clg key (3).

    >>> c = Config('hi: there')
    >>> c.hi
    'there'

    >>> import os
    >>> tmpfile = '/tmp/test_tempfile.conf'
    >>> with open(tmpfile, 'w') as fh:
    ...     ret = fh.write('a: 1, b: 2')
    >>> c = Config(args=['', '-C={}'.format(tmpfile)])
    >>> c.b
    2
    >>> os.remove(tmpfile)

    >>> conf = """
    ...    clg:
    ...        description: Build a full system
    ...        args:
    ...            host:
    ...                help: Host to build"""
    >>> c = Config(args=['', '-E={}'.format(conf), 'leon'])
    >>> c.host
    'leon'
    '''
    # Try up to max times to expand $ keys
    expand_max = 5

    def __init__(self, config_data='', args=None, version=None, types=set()):
        '''Initialize config object. Keep its __dict__ clean for easy access'''
        super(Config, self).__init__()
        if config_data == '' and args is None:
            return
        if version:
            self.version = version
        if args and 'clg' in config_data:
            self.prog = args[0]
        self._expand_keys(config_data)
        args = self._load_options(args)
        self._load_config_cli(args, types)
        self._checkconfig()

    def _expand_keys(self, config_data=''):
        '''Add config_data into config and interpolate $keys.

        >>> config_data = 'data_path: /data, data_file: $data_path/data.txt'
        >>> c = Config()
        >>> c._expand_keys(config_data)
        >>> c.data_file
        '/data/data.txt'
        '''
        self.update(config_data)
        # Convert self to text for full key interpolation
        config_string = str(self)
        n = count()
        while '$' in config_string and next(n) < self.expand_max:
            config_string = self.render(config_string)
            self.update(config_string)

    def _load_config_file(self, filepath):
        '''Return config file adding data from loadconfig.template keyword'''
        datafile = read_config_file(filepath)
        self._expand_keys(datafile)
        return datafile

    def _load_options(self, args):
        '''Load config from options -E and -C from cli arguments.

        >>> c = Config()
        >>> c._load_options(['-E="data_file: data.txt"'])
        []
        >>> c.data_file
        'data.txt'
        '''
        if args is None:
            return
        for arg in findregex('^(-E|-C)=', args):
            config_string, option = _get_option(arg)
            if option == 'C':
                # Expand $ keys in file name
                config_string = self.render(config_string)
                config_string = self._load_config_file(config_string)
            if 'prog' not in self and 'clg' in config_string:
                self.prog = args[0]
            self._expand_keys(config_string)
        # Prevent clg seeing -E or -C options
        return delregex('^(-E|-C)=', args)

    def _load_config_cli(self, args, types=set()):
        '''Load config parsing cli arguments with clg.
        clg key config may come from config file, cli arg, or python config.

        >>> from os.path import basename
        >>> c = Config()
        >>> c.update("""
        ...     clg:
        ...         args:
        ...             filename:
        ...                 type: basename
        ...     """)
        >>> c._load_config_cli(args=['', '/data/data.txt'], types={basename})
        >>> c.filename
        'data.txt'
        '''
        if not args or 'clg' not in self:
            return
        clg_args = _clg_parse(self.clg, args, types)
        self._expand_keys(clg_args)  # Add config from cli args
        del self['clg']  # Remove clg key from Config

    def _checkconfig(self):
        '''Check configuration using keyword checkconfig

        >>> c = Config("""
        ...         action: stop
        ...         checkconfig: |
        ...             if  '$action' == 'stop':
        ...                 raise Exception('Stopping now.')
        ...         """)
        Traceback (most recent call last):
            ...
        Exception: Stopping now.
        '''
        if 'checkconfig' in self:
            exec(self.checkconfig)
            self._expand_keys('')
            del self['checkconfig']

    def export(self):
        '''Export the config for shell usage.
        Keys are uppercased. List-like keys are flattened.

        >>> c = Config('activity: hanggliding')
        >>> c.export()
        'export ACTIVITY="hanggliding"'
        '''
        retval = ''
        for key in self:
            val = dfl(self[key])
            # Make list-like keys shell friendly
            if isinstance(val, (list, tuple)):
                val = flatten(val)
                val = ' '.join([shlex_quote(e) for e in val])
            elif isinstance(val, dict):
                val = repr(val)
            retval += 'export {}="{}"\n'.format(
                key.upper().replace(' ', '_'), val)
        return retval[:-1]

    def render(self, template):
        '''Render a string template

        >>> c = Config('name: Jay')
        >>> c.render('Good morning $name.')
        'Good morning Jay.'
        '''
        return Template(template).safe_substitute(self)

    def run(self, module='__main__'):
        r'''Run selected subparser command.

        >>> conf = Config("""
        ...            prog: netapplet.py
        ...            clg:
        ...                subparsers:
        ...                    install:
        ...                        help: 'run as $prog install | bash'""")
        >>> def install(c):
        ...     print('echo "Installation shell script lines for {}."'.format(
        ...         c.prog))
        >>> # Ensure install function is reachable
        >>> sys.modules[__name__].install = install
        >>> c = Config(conf, args=['', 'install'])
        >>> c.run(__name__)
        echo "Installation shell script lines for netapplet.py."
        '''
        if isinstance(module, str):
            module = sys.modules[module]
        getattr(module, str(self.command0))(self)

    def __repr__(self):
        return repr(Odict(self))

    def __str__(self):
        return str(Odict(self))
<commit_msg>dev version update<commit_after>'''loadconfig python library'''
from __future__ import print_function
__all__ = ['Config', 'Odict', '__version__']

__author__ = 'Daniel Mizyrycki'
__version__ = '0.1.1.dev0'

from os.path import dirname, abspath
import sys
sys.path.insert(0, dirname(abspath(__file__)))

from collections import OrderedDict
from copy import deepcopy
from itertools import count
from lib import (delregex, dfl, findregex, flatten, read_config_file,
    read_file, _clg_parse, _get_option)
from os import environ
from py6 import cStringIO, shlex_quote
from string import Template
import yaml
from yaml.scanner import ScannerError

class Odict(OrderedDict):
    r'''Add more readable representation to OrderedDict using yaml.

    >>> d = Odict('{a: 1, b: {c: 3}}')
    >>> repr(d)
    '{a: 1, b: {c: 3}}'
    >>> str(d)
    'a: 1\nb:\n  c: 3'
    >>> d == Odict(d)
    True
    >>> d.b.c
    3
    >>> d._p
    a: 1
    b:
      c: 3
    '''
    def __init__(self, *args, **kwargs):
        '''Process first argument as yaml if it is a string'''
        args = self._process_args(*args, **kwargs)
        super(Odict, self).__init__(*args, **kwargs)

    def _process_args(self, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], str):
            try:
                pair_list = self.load(args[0])
            except ScannerError:
                # Yaml needs {} on multi-key single-line strings
                pair_list = self.load('{{{}}}'.format(args[0]))
            args = [pair_list] if pair_list else []
        return args

    def update(self, *args, **kwargs):
        '''Add yaml string as posible argument

        >>> d = Odict('hi: there')
        >>> d.hi
        'there'
        >>> d.update({'hi': 'Liss'})
        >>> d.hi
        'Liss'
        '''
        args = self._process_args(*args, **kwargs)
        super(Odict, self).update(*args, **kwargs)
        # Remove '_' key possible used by include
        if '_' in self:
            del self['_']

    def __getattr__(self, name):
        '''Access Odict key as attribute

        >>> c = Config('activity: [hike, bike, scuba dive, run]')
        >>> c.activity
        ['hike', 'bike', 'scuba dive', 'run']
        '''
        if name in self:
            return self[name]
        if name in ['_OrderedDict__root', '__deepcopy__']:
            return super(Odict, self).__getattr__(name)

    def __delattr__(self, name):
        '''Delete Odict key from attribute
        >>> c = Config('activity: [hike, bike, scuba dive, run]')
        >>> del c.activity
        >>> c
        {}
        '''
        del self[name]

    def __setattr__(self, name, value):
        '''Set Odict key from attribute

        >>> c = Config('activity: [hike, scuba dive]')
        >>> c.place = 'Hawaii'
        >>> c
        {activity: [hike, scuba dive], place: Hawaii}
        '''
        if name in ['_OrderedDict__root', '_OrderedDict__hardroot',
         '_OrderedDict__map']:
            return super(Odict, self).__setattr__(name, value)
        self[name] = value

    def __str__(self):
        return Odict.dump(self, False)

    def __repr__(self):
        return Odict.dump(self)

    # Convenient shortcuts
    _r = property(__repr__)
    _s = property(__str__)
    _p = property(lambda x: print(str(x)))

    @staticmethod
    def load(yaml_string):
        '''Return pair list from a yaml string'''
        class Loader(yaml.SafeLoader):
            yaml_mapping = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
            _data = ''

            def __init__(self, *args, **kwargs):
                super(Loader, self).__init__(*args, **kwargs)
                self.add_constructor(self.yaml_mapping, self.construct_mapping)
                self.add_constructor('!env', self.env)
                self.add_constructor('!read', self.read)
                self.add_constructor('!include', self.include)
                self.add_constructor('!expand', self.expand)

            def env(self, safeloader, node):
                node = self.construct_scalar(node)
                if node.upper() in environ:
                    return {node: environ[node.upper()]}
                return {node: ''}

            def read(self, safeloader, node):
                node = self.construct_scalar(node)
                return read_file(node)

            def include(self, safeloader, node):
                node = self.construct_scalar(node)
                filepath, sep, key = node.partition(':')
                self._data = yaml.load(read_file(filepath), Loader)
                return self.subkey(key)

            def expand(self, safeloader, node):
                key = self.construct_scalar(node)
                self._data = safeloader._data
                return self.subkey(key)

            def subkey(self, key):
                if not self._data:
                    return ''
                data = deepcopy(self._data)
                if key == '&':
                    return ''
                for k in key.split(':'):
                    if not k:
                        return data
                    data = data.get(k)
                    if data is None:
                        return ''
                return data

            def construct_mapping(self, safeloader, node):
                safeloader.flatten_mapping(node)
                return Odict(safeloader.construct_pairs(node))

        return yaml.load(yaml_string, Loader)

    @staticmethod
    def dump(self, default_flow_style=True):
        '''Serialize odict into yaml string'''
        class Dumper(yaml.SafeDumper):
            def __init__(self, *args, **kwargs):
                super(Dumper, self).__init__(*args, **kwargs)
                self.ignore_aliases = lambda self: True
                self.add_representer(Odict, lambda self, data:
                    self.represent_dict(data.items()))

            def increase_indent(self, flow=False, indentless=False):
                return super(Dumper, self).increase_indent(flow, False)

        stream = cStringIO()
        yaml.dump(self, stream, Dumper, default_flow_style=default_flow_style)
        return stream.getvalue()[:-1]

class Config(Odict):
    '''Config class for programs.

    The Config class is designed to be a versatile interface to load a config
    object for use in python and shell programs.
    Source data comes from config_data object and from interpreting the args
    parameter (usually sys.args) from command line.
    config_data object can be a yaml string or a dictionary (eg: 1 doctest).
    args list take any number of arguments. Optionally some arguments can be
    in the form of -C (for config file) and -E (for extra) options (2).
    The python clg package is used to parse normal cli arguments using the
    special clg key (3).

    >>> c = Config('hi: there')
    >>> c.hi
    'there'

    >>> import os
    >>> tmpfile = '/tmp/test_tempfile.conf'
    >>> with open(tmpfile, 'w') as fh:
    ...     ret = fh.write('a: 1, b: 2')
    >>> c = Config(args=['', '-C={}'.format(tmpfile)])
    >>> c.b
    2
    >>> os.remove(tmpfile)

    >>> conf = """
    ...    clg:
    ...        description: Build a full system
    ...        args:
    ...            host:
    ...                help: Host to build"""
    >>> c = Config(args=['', '-E={}'.format(conf), 'leon'])
    >>> c.host
    'leon'
    '''
    # Try up to max times to expand $ keys
    expand_max = 5

    def __init__(self, config_data='', args=None, version=None, types=set()):
        '''Initialize config object. Keep its __dict__ clean for easy access'''
        super(Config, self).__init__()
        if config_data == '' and args is None:
            return
        if version:
            self.version = version
        if args and 'clg' in config_data:
            self.prog = args[0]
        self._expand_keys(config_data)
        args = self._load_options(args)
        self._load_config_cli(args, types)
        self._checkconfig()

    def _expand_keys(self, config_data=''):
        '''Add config_data into config and interpolate $keys.

        >>> config_data = 'data_path: /data, data_file: $data_path/data.txt'
        >>> c = Config()
        >>> c._expand_keys(config_data)
        >>> c.data_file
        '/data/data.txt'
        '''
        self.update(config_data)
        # Convert self to text for full key interpolation
        config_string = str(self)
        n = count()
        while '$' in config_string and next(n) < self.expand_max:
            config_string = self.render(config_string)
            self.update(config_string)

    def _load_config_file(self, filepath):
        '''Return config file adding data from loadconfig.template keyword'''
        datafile = read_config_file(filepath)
        self._expand_keys(datafile)
        return datafile

    def _load_options(self, args):
        '''Load config from options -E and -C from cli arguments.

        >>> c = Config()
        >>> c._load_options(['-E="data_file: data.txt"'])
        []
        >>> c.data_file
        'data.txt'
        '''
        if args is None:
            return
        for arg in findregex('^(-E|-C)=', args):
            config_string, option = _get_option(arg)
            if option == 'C':
                # Expand $ keys in file name
                config_string = self.render(config_string)
                config_string = self._load_config_file(config_string)
            if 'prog' not in self and 'clg' in config_string:
                self.prog = args[0]
            self._expand_keys(config_string)
        # Prevent clg seeing -E or -C options
        return delregex('^(-E|-C)=', args)

    def _load_config_cli(self, args, types=set()):
        '''Load config parsing cli arguments with clg.
        clg key config may come from config file, cli arg, or python config.

        >>> from os.path import basename
        >>> c = Config()
        >>> c.update("""
        ...     clg:
        ...         args:
        ...             filename:
        ...                 type: basename
        ...     """)
        >>> c._load_config_cli(args=['', '/data/data.txt'], types={basename})
        >>> c.filename
        'data.txt'
        '''
        if not args or 'clg' not in self:
            return
        clg_args = _clg_parse(self.clg, args, types)
        self._expand_keys(clg_args)  # Add config from cli args
        del self['clg']  # Remove clg key from Config

    def _checkconfig(self):
        '''Check configuration using keyword checkconfig

        >>> c = Config("""
        ...         action: stop
        ...         checkconfig: |
        ...             if  '$action' == 'stop':
        ...                 raise Exception('Stopping now.')
        ...         """)
        Traceback (most recent call last):
            ...
        Exception: Stopping now.
        '''
        if 'checkconfig' in self:
            exec(self.checkconfig)
            self._expand_keys('')
            del self['checkconfig']

    def export(self):
        '''Export the config for shell usage.
        Keys are uppercased. List-like keys are flattened.

        >>> c = Config('activity: hanggliding')
        >>> c.export()
        'export ACTIVITY="hanggliding"'
        '''
        retval = ''
        for key in self:
            val = dfl(self[key])
            # Make list-like keys shell friendly
            if isinstance(val, (list, tuple)):
                val = flatten(val)
                val = ' '.join([shlex_quote(e) for e in val])
            elif isinstance(val, dict):
                val = repr(val)
            retval += 'export {}="{}"\n'.format(
                key.upper().replace(' ', '_'), val)
        return retval[:-1]

    def render(self, template):
        '''Render a string template

        >>> c = Config('name: Jay')
        >>> c.render('Good morning $name.')
        'Good morning Jay.'
        '''
        return Template(template).safe_substitute(self)

    def run(self, module='__main__'):
        r'''Run selected subparser command.

        >>> conf = Config("""
        ...            prog: netapplet.py
        ...            clg:
        ...                subparsers:
        ...                    install:
        ...                        help: 'run as $prog install | bash'""")
        >>> def install(c):
        ...     print('echo "Installation shell script lines for {}."'.format(
        ...         c.prog))
        >>> # Ensure install function is reachable
        >>> sys.modules[__name__].install = install
        >>> c = Config(conf, args=['', 'install'])
        >>> c.run(__name__)
        echo "Installation shell script lines for netapplet.py."
        '''
        if isinstance(module, str):
            module = sys.modules[module]
        getattr(module, str(self.command0))(self)

    def __repr__(self):
        return repr(Odict(self))

    def __str__(self):
        return str(Odict(self))
<|endoftext|>


<commit_before>from collections import defaultdict
import os
import subprocess
import yaml
import multiprocessing
import gzip
import jobit
import pickle
import logging

from IMProv.IMProvDoc import IMProvDoc
from ProdCommon.FwkJobRep.ReportParser import readJobReport

from lobster import job, util
import jobit
import dash

def resolve_name(job, merged_job, name, name_format):
    base, ext = os.path.splitext(name)
    id = str(job) if merged_job == 0 else 'merged_{0}'.format(merged_job)

    return name_format.format(base=base, ext=ext[1:], id=id)

class MergeHandler(object):
    def __init__(self, id, dataset, chirp, jobdir, outname, num_outputs, outname_index, sdir, jobs):
        self.__id = id
        self.__dataset = dataset
        self.__chirp = chirp
        self.__jobdir = jobdir
        self.__outname = outname
        self.__sdir = sdir
        self.__jobs = jobs

        self.__reports = []
        self.__inputs = []

        if num_outputs == 1:
            self.__tag = str(id)
            self.__base = ''
        else:
            base, ext = os.path.splitext(outname)
            self.__tag = '{0}_{1}'.format(id, outname_index)
            self.__base = '{0}_'.format(base)

    @property
    def dataset(self):
        return self.__dataset

    @property
    def jobs(self):
        return self.__jobs

    @property
    def reports(self):
        return self.__reports

    @property
    def id(self):
        return self.__id

    @property
    def inputs(self):
        return self.__inputs

    @property
    def jobdir(self):
        return self.__jobdir

    @property
    def outname(self):
        return self.__outname

    @property
    def base(self):
        return self.__base

    @property
    def tag(self):
        return self.__tag

    def get_job_update(self, failed, outsize):
        if failed:
            status = jobit.FAILED
        else:
            status = jobit.SUCCESSFUL

        return [(job, self.__id, status, outsize) for job in self.__jobs]

    def get_job_info(self):
        args = ['output=' + self.__outname]
        if self.__chirp:
            args += ['chirp={0}'.format(self.__chirp)]
            args += [str('inputs=' + ','.join(self.__inputs))]

        files = ['file:' + os.path.basename(x) for x in self.__inputs]

        return args, files

    def merge_reports(self):
        reports = []
        for r in self.__reports:
            f = gzip.open(r)
            report = readJobReport(f)
            reports.extend(report)
            f.close()

        output = IMProvDoc("JobReports")
        for item in reports:
            output.addNode(item.save())

        outfile = gzip.open(os.path.join(self.__jobdir, 'report.xml.gz'), 'wb')
        outfile.write(output.makeDOMDocument().toprettyxml())
        outfile.close()

    def cleanup(self):
#         tdir = os.path.dirname(self.__jobdir) # FIXME Do we want to delete old report.xmls?
#         for r in self.__reports:
#             os.remove(r)
        for file in self.__inputs:
            fullpath = os.path.join(self.__sdir, os.path.basename(file))
            os.remove(fullpath)

class MergeProvider(job.JobProvider):
    def __init__(self, config):
        super(MergeProvider, self).__init__(config)

        self.__chirp = self.config.get('stageout server', None)
        self.__sandbox = os.path.join(self.workdir, 'sandbox')
        self.__dash = dash.DummyMonitor(self.taskid)
        self.__mergehandlers = {}

        self.__store = jobit.JobitStore(self.config)
        self.__store.reset_merging()
        logging.info("registering unmerged jobs")
        self.__store.register_unmerged()

        self.__grid_files = [(os.path.join('/cvmfs/grid.cern.ch', x), os.path.join('grid', x)) for x in
                                 ['3.2.11-1/external/etc/profile.d/clean-grid-env-funcs.sh',
                                  '3.2.11-1/external/etc/profile.d/grid-env-funcs.sh',
                                  '3.2.11-1/external/etc/profile.d/grid-env.sh',
                                  '3.2.11-1/etc/profile.d/grid-env.sh',
                                  '3.2.11-1/glite/bin/voms-proxy-info',
                                  '3.2.11-1/glite/lib64/libvomsapi_nog.so.0.0.0',
                                  '3.2.11-1/glite/lib64/libvomsapi_nog.so.0',
                                  'etc/grid-security/certificates'
                                  ]
                             ]

        self.__common_inputs = [(self.__sandbox + ".tar.bz2", "sandbox.tar.bz2"),
                                (os.path.join(os.path.dirname(__file__), 'data', 'mtab'), 'mtab'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'siteconfig'), 'siteconfig'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'wrapper.sh'), 'wrapper.sh'),
                                (self.parrot_bin, 'bin'),
                                (self.parrot_lib, 'lib'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'job.py'), 'job.py'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'merge_cfg.py'), 'merge_cfg.py')
                                ] + self.__grid_files

        if not util.checkpoint(self.workdir, 'sandbox'):
            raise NotImplementedError

    def get_report(self, label, job, merge_job):
        if merge_job == 0:
            jobdir = self.get_jobdir(job, label, 'successful')
        else:
            jobdir = self.get_jobdir(merge_job, label, 'merged')
        return os.path.join(jobdir, 'report.xml.gz')

    def obtain(self, num=1):
        unmerged_jobs = self.retry(self.__store.pop_unmerged_jobs, (num,), {})
        if not unmerged_jobs or len(unmerged_jobs) == 0:
            return None

        tasks = []
        for merging_job, dset, jobs in unmerged_jobs:
            out_tag = 'merged_{0}'.format(merging_job)

            monitorid, syncid = self.__dash.register_job(merging_job)

            sdir = os.path.join(self.stageout, dset)
            jdir = self.create_jobdir(merging_job, dset, 'merging')

            for outname_index, local_outname in enumerate(self.outputs[dset]):
                base, ext = os.path.splitext(local_outname)
                remote_outname = self.outputformats[dset].format(base=base, ext=ext[1:], id=out_tag)

                handler = MergeHandler(merging_job,
                                       dset,
                                       self.__chirp,
                                       jdir,
                                       local_outname,
                                       len(self.outputs[dset]),
                                       outname_index,
                                       sdir,
                                       [job for job, merged_job in jobs])

                stageout = [(local_outname, os.path.join(dset, remote_outname))]
                outputs = [(os.path.join(jdir, '{0}{1}'.format(handler.base, f)), f) for f in ['cmssw.log.gz', 'report.pkl']]
                if not self.__chirp:
                    outputs.append((os.path.join(sdir, remote_outname), local_outname))

                inputs = self.__common_inputs[:]
                if 'X509_USER_PROXY' in os.environ:
                    inputs.append((os.environ['X509_USER_PROXY'], 'proxy'))

                for job, merged_job in jobs:
                    handler.reports.append(self.get_report(dset, job, merged_job))

                    input = resolve_name(job, merged_job, local_outname, self.outputformats[dset])
                    handler.inputs.append(os.path.join(os.path.basename(sdir), input))

                    if not self.__chirp:
                        inputs.append((os.path.join(sdir, input), input))

                args, files = handler.get_job_info()
                with open(os.path.join(jdir, 'parameters.pkl'), 'wb') as f:
                    pickle.dump((args, files, None, stageout, self.__chirp, self.taskid, monitorid, syncid, True), f, pickle.HIGHEST_PROTOCOL)
                inputs.append((os.path.join(jdir, 'parameters.pkl'), 'parameters.pkl'))

                cmd = 'sh wrapper.sh python job.py merge_cfg.py parameters.pkl'

                tasks.append((handler.tag, cmd, inputs, outputs))

                self.__mergehandlers[handler.tag] = handler

                resolved_ids = [str(job) if merged_job==0 else str(merged_job) for job, merged_job in jobs]
                logging.info("creating task {0} to merge jobs {1}".format(handler.tag, ", ".join(resolved_ids)))

        return tasks

    def release(self, tasks):
        jobs = []
        for task in tasks:
            failed = task.return_status != 0

            handler = self.__mergehandlers[task.tag]

            if task.output:
                f = gzip.open(os.path.join(handler.jobdir, '{0}job.log.gz'.format(handler.base)), 'wb')
                f.write(task.output)
                f.close()

            files_info = {}
            files_skipped = []
            events_written = 0
            task_times = [None] * 7
            cmssw_exit_code = None
            cputime = 0
            outsize = 0

            try:
                with open(os.path.join(handler.jobdir, '{0}report.pkl'.format(handler.base)), 'rb') as f:
                    files_info, files_skipped, events_written, task_times, cmssw_exit_code, cputime, outsize = pickle.load(f)
            except (EOFError, IOError) as e:
                failed = True
                logging.error("error processing {0}:\n{1}".format(task.tag, e))

            if cmssw_exit_code not in (None, 0):
                exit_code = cmssw_exit_code
                if exit_code > 0:
                    failed = True
            else:
                exit_code = task.return_status

            jobs += handler.get_job_update(failed, outsize)
            if failed:
                self.move_jobdir(handler.id, handler.dataset, 'merge_failed', 'merging')
            else:
                handler.merge_reports()
                handler.cleanup()
                self.move_jobdir(handler.id, handler.dataset, 'merged', 'merging')

            logging.info("job {0} returned with exit code {1}".format(task.tag, exit_code))

            del self.__mergehandlers[task.tag]

        if len(jobs) > 0:
            self.retry(self.__store.update_merged, (jobs,), {})

    def done(self):
        return self.__store.unfinished_merging() == 0

    def work_left(self):
        return self.__store.unfinished_merging()
<commit_msg>Use max megabytes to set merged file size.<commit_after>from collections import defaultdict
import os
import subprocess
import yaml
import multiprocessing
import gzip
import jobit
import pickle
import logging

from IMProv.IMProvDoc import IMProvDoc
from ProdCommon.FwkJobRep.ReportParser import readJobReport

from lobster import job, util
import jobit
import dash

def resolve_name(job, merged_job, name, name_format):
    base, ext = os.path.splitext(name)
    id = str(job) if merged_job == 0 else 'merged_{0}'.format(merged_job)

    return name_format.format(base=base, ext=ext[1:], id=id)

class MergeHandler(object):
    def __init__(self, id, dataset, chirp, jobdir, outname, num_outputs, outname_index, sdir, jobs):
        self.__id = id
        self.__dataset = dataset
        self.__chirp = chirp
        self.__jobdir = jobdir
        self.__outname = outname
        self.__sdir = sdir
        self.__jobs = jobs

        self.__reports = []
        self.__inputs = []

        if num_outputs == 1:
            self.__tag = str(id)
            self.__base = ''
        else:
            base, ext = os.path.splitext(outname)
            self.__tag = '{0}_{1}'.format(id, outname_index)
            self.__base = '{0}_'.format(base)

    @property
    def dataset(self):
        return self.__dataset

    @property
    def jobs(self):
        return self.__jobs

    @property
    def reports(self):
        return self.__reports

    @property
    def id(self):
        return self.__id

    @property
    def inputs(self):
        return self.__inputs

    @property
    def jobdir(self):
        return self.__jobdir

    @property
    def outname(self):
        return self.__outname

    @property
    def base(self):
        return self.__base

    @property
    def tag(self):
        return self.__tag

    def get_job_update(self, failed, outsize):
        if failed:
            status = jobit.FAILED
        else:
            status = jobit.SUCCESSFUL

        return [(job, self.__id, status, outsize) for job in self.__jobs]

    def get_job_info(self):
        args = ['output=' + self.__outname]
        if self.__chirp:
            args += ['chirp={0}'.format(self.__chirp)]
            args += [str('inputs=' + ','.join(self.__inputs))]

        files = ['file:' + os.path.basename(x) for x in self.__inputs]

        return args, files

    def merge_reports(self):
        reports = []
        for r in self.__reports:
            f = gzip.open(r)
            report = readJobReport(f)
            reports.extend(report)
            f.close()

        output = IMProvDoc("JobReports")
        for item in reports:
            output.addNode(item.save())

        outfile = gzip.open(os.path.join(self.__jobdir, 'report.xml.gz'), 'wb')
        outfile.write(output.makeDOMDocument().toprettyxml())
        outfile.close()

    def cleanup(self):
#         tdir = os.path.dirname(self.__jobdir) # FIXME Do we want to delete old report.xmls?
#         for r in self.__reports:
#             os.remove(r)
        for file in self.__inputs:
            fullpath = os.path.join(self.__sdir, os.path.basename(file))
            os.remove(fullpath)

class MergeProvider(job.JobProvider):
    def __init__(self, config):
        super(MergeProvider, self).__init__(config)

        self.__chirp = self.config.get('stageout server', None)
        self.__sandbox = os.path.join(self.workdir, 'sandbox')
        self.__dash = dash.DummyMonitor(self.taskid)
        self.__mergehandlers = {}

        self.__store = jobit.JobitStore(self.config)
        self.__store.reset_merging()
        logging.info("registering unmerged jobs")
        self.__store.register_unmerged(config.get('max megabytes', 3500))

        self.__grid_files = [(os.path.join('/cvmfs/grid.cern.ch', x), os.path.join('grid', x)) for x in
                                 ['3.2.11-1/external/etc/profile.d/clean-grid-env-funcs.sh',
                                  '3.2.11-1/external/etc/profile.d/grid-env-funcs.sh',
                                  '3.2.11-1/external/etc/profile.d/grid-env.sh',
                                  '3.2.11-1/etc/profile.d/grid-env.sh',
                                  '3.2.11-1/glite/bin/voms-proxy-info',
                                  '3.2.11-1/glite/lib64/libvomsapi_nog.so.0.0.0',
                                  '3.2.11-1/glite/lib64/libvomsapi_nog.so.0',
                                  'etc/grid-security/certificates'
                                  ]
                             ]

        self.__common_inputs = [(self.__sandbox + ".tar.bz2", "sandbox.tar.bz2"),
                                (os.path.join(os.path.dirname(__file__), 'data', 'mtab'), 'mtab'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'siteconfig'), 'siteconfig'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'wrapper.sh'), 'wrapper.sh'),
                                (self.parrot_bin, 'bin'),
                                (self.parrot_lib, 'lib'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'job.py'), 'job.py'),
                                (os.path.join(os.path.dirname(__file__), 'data', 'merge_cfg.py'), 'merge_cfg.py')
                                ] + self.__grid_files

        if not util.checkpoint(self.workdir, 'sandbox'):
            raise NotImplementedError

    def get_report(self, label, job, merge_job):
        if merge_job == 0:
            jobdir = self.get_jobdir(job, label, 'successful')
        else:
            jobdir = self.get_jobdir(merge_job, label, 'merged')
        return os.path.join(jobdir, 'report.xml.gz')

    def obtain(self, num=1):
        unmerged_jobs = self.retry(self.__store.pop_unmerged_jobs, (num,), {})
        if not unmerged_jobs or len(unmerged_jobs) == 0:
            return None

        tasks = []
        for merging_job, dset, jobs in unmerged_jobs:
            out_tag = 'merged_{0}'.format(merging_job)

            monitorid, syncid = self.__dash.register_job(merging_job)

            sdir = os.path.join(self.stageout, dset)
            jdir = self.create_jobdir(merging_job, dset, 'merging')

            for outname_index, local_outname in enumerate(self.outputs[dset]):
                base, ext = os.path.splitext(local_outname)
                remote_outname = self.outputformats[dset].format(base=base, ext=ext[1:], id=out_tag)

                handler = MergeHandler(merging_job,
                                       dset,
                                       self.__chirp,
                                       jdir,
                                       local_outname,
                                       len(self.outputs[dset]),
                                       outname_index,
                                       sdir,
                                       [job for job, merged_job in jobs])

                stageout = [(local_outname, os.path.join(dset, remote_outname))]
                outputs = [(os.path.join(jdir, '{0}{1}'.format(handler.base, f)), f) for f in ['cmssw.log.gz', 'report.pkl']]
                if not self.__chirp:
                    outputs.append((os.path.join(sdir, remote_outname), local_outname))

                inputs = self.__common_inputs[:]
                if 'X509_USER_PROXY' in os.environ:
                    inputs.append((os.environ['X509_USER_PROXY'], 'proxy'))

                for job, merged_job in jobs:
                    handler.reports.append(self.get_report(dset, job, merged_job))

                    input = resolve_name(job, merged_job, local_outname, self.outputformats[dset])
                    handler.inputs.append(os.path.join(os.path.basename(sdir), input))

                    if not self.__chirp:
                        inputs.append((os.path.join(sdir, input), input))

                args, files = handler.get_job_info()
                with open(os.path.join(jdir, 'parameters.pkl'), 'wb') as f:
                    pickle.dump((args, files, None, stageout, self.__chirp, self.taskid, monitorid, syncid, True), f, pickle.HIGHEST_PROTOCOL)
                inputs.append((os.path.join(jdir, 'parameters.pkl'), 'parameters.pkl'))

                cmd = 'sh wrapper.sh python job.py merge_cfg.py parameters.pkl'

                tasks.append((handler.tag, cmd, inputs, outputs))

                self.__mergehandlers[handler.tag] = handler

                resolved_ids = [str(job) if merged_job==0 else str(merged_job) for job, merged_job in jobs]
                logging.info("creating task {0} to merge jobs {1}".format(handler.tag, ", ".join(resolved_ids)))

        return tasks

    def release(self, tasks):
        jobs = []
        for task in tasks:
            failed = task.return_status != 0

            handler = self.__mergehandlers[task.tag]

            if task.output:
                f = gzip.open(os.path.join(handler.jobdir, '{0}job.log.gz'.format(handler.base)), 'wb')
                f.write(task.output)
                f.close()

            files_info = {}
            files_skipped = []
            events_written = 0
            task_times = [None] * 7
            cmssw_exit_code = None
            cputime = 0
            outsize = 0

            try:
                with open(os.path.join(handler.jobdir, '{0}report.pkl'.format(handler.base)), 'rb') as f:
                    files_info, files_skipped, events_written, task_times, cmssw_exit_code, cputime, outsize = pickle.load(f)
            except (EOFError, IOError) as e:
                failed = True
                logging.error("error processing {0}:\n{1}".format(task.tag, e))

            if cmssw_exit_code not in (None, 0):
                exit_code = cmssw_exit_code
                if exit_code > 0:
                    failed = True
            else:
                exit_code = task.return_status

            jobs += handler.get_job_update(failed, outsize)
            if failed:
                self.move_jobdir(handler.id, handler.dataset, 'merge_failed', 'merging')
            else:
                handler.merge_reports()
                handler.cleanup()
                self.move_jobdir(handler.id, handler.dataset, 'merged', 'merging')

            logging.info("job {0} returned with exit code {1}".format(task.tag, exit_code))

            del self.__mergehandlers[task.tag]

        if len(jobs) > 0:
            self.retry(self.__store.update_merged, (jobs,), {})

    def done(self):
        return self.__store.unfinished_merging() == 0

    def work_left(self):
        return self.__store.unfinished_merging()
<|endoftext|>


<commit_before>import elasticsearch as es
import elasticsearch_dsl as es_dsl
from datetime import datetime
import json
import re
import inspect
import logging

from lobster.util import Configurable

logger = logging.getLogger('lobster.monitor.elk')

class ElkInterface(Configurable):
    """
    Enables ELK stack monitoring for the current Lobster run using an existing
    Elasticsearch instance.

    Parameters
    ----------
        host : str
            Host running Elasticsearch cluster.
        port : int
            Port number running Elasticsearch HTTP service.
        user : str
            User ID to label Elasticsearch indices and Kibana objects.
        project : str
            Project ID to label Elasticsearch indices and Kibana objects.
        modules : list
            List of modules to include from the Kibana templates. Defaults to
            including only the core templates.
        populate_template : bool
            Whether to send documents to indices with the [template] prefix,
            in addition to sending them to the [user_run] prefix. Defaults to
            false.
    """
    _mutable = {}

    def __init__(self, host, port, user, project, modules=None,
                 populate_template=False):
        self.host = host
        self.port = port
        self.user = user
        self.project = project
        self.modules = modules or ['core']
        self.populate_template = populate_template
        self.prefix = '[' + self.user + '_' + self.project + ']'
        self.client = es.Elasticsearch([{'host': self.host,
                                         'port': self.port}])

    def __getstate__(self):
        state = {'host': self.host,
                 'port': self.port,
                 'user': self.user,
                 'project': self.project,
                 'modules': self.modules,
                 'prefix': self.prefix}
        return state

    def __setstate__(self, state):
        self.host = state['host']
        self.port = state['port']
        self.user = state['user']
        self.project = state['project']
        self.modules = state['modules']
        self.prefix = state['prefix']
        self.client = es.Elasticsearch([{'host': self.host,
                                         'port': self.port}])

    def check_prefix(self):
        logger.info("checking Elasticsearch and Kibana prefixes")

        try:
            indices = self.client.indices.get_aliases().keys()
        except es.exceptions.ConnectionError as e:
            logger.error(e)
            raise e

        if any(self.prefix in s for s in indices):
            e = AttributeError("Elasticsearch indices with prefix " +
                               self.prefix + " already exist.")
            logger.error(e)
            raise e

        search = es_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=self.prefix)
        response = search.execute()
        if len(response) > 0:
            e = AttributeError("Kibana objects with prefix " +
                               self.prefix + " already exist.")
            logger.error(e)
            raise e

    def generate_kibana_objects(self):
        logger.info("generating Kibana objects from templates")

        temp_prefix = '[template]'
        self.prefix = '[' + self.user + '_' + self.project + ']'
        other_prefix = re.compile('\[.*\]')

        search_index = es_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=temp_prefix) \
            .filter('match', _type='index-pattern')
        response_index = search_index.execute()

        for index in response_index:
            index.meta.id = index.meta.id.replace(temp_prefix, self.prefix, 1)
            index.title = index.title.replace(temp_prefix, self.prefix, 1)
            self.client.index(index='.kibana', doc_type=index.meta.doc_type,
                              id=index.meta.id, body=index.to_dict())

        for module in self.modules:
            temp_mod_prefix = temp_prefix + '[' + module + ']'
            new_mod_prefix = self.prefix + '[' + module + ']'

            search_vis = es_dsl.Search(using=self.client, index='.kibana') \
                .filter('prefix', _id=temp_mod_prefix) \
                .filter('match', _type='visualization')

            search_dash = es_dsl.Search(using=self.client, index='.kibana') \
                .filter('prefix', _id=temp_mod_prefix) \
                .filter('match', _type='dashboard')

            response_vis = search_vis.execute()
            response_dash = search_dash.execute()

            for vis in response_vis:
                vis.meta.id = vis.meta.id.replace(
                    temp_mod_prefix, new_mod_prefix, 1)
                vis.title = vis.title.replace(
                    temp_mod_prefix, new_mod_prefix, 1)

                source = json.loads(vis.kibanaSavedObjectMeta.searchSourceJSON)
                source['index'] = other_prefix.sub(
                    self.prefix, source['index'])
                vis.kibanaSavedObjectMeta.searchSourceJSON = json.dumps(source)

                self.client.index(index='.kibana', doc_type=vis.meta.doc_type,
                                  id=vis.meta.id, body=vis.to_dict())

            for dash in response_dash:
                dash.meta.id = dash.meta.id.replace(
                    temp_mod_prefix, new_mod_prefix, 1)
                dash.title = dash.title.replace(
                    temp_mod_prefix, new_mod_prefix, 1)

                dash_panels = json.loads(dash.panelsJSON)
                for panel in dash_panels:
                    panel['id'] = panel['id'].replace(
                        temp_mod_prefix, new_mod_prefix, 1)
                dash.panelsJSON = json.dumps(dash_panels)

                self.client.index(index='.kibana', doc_type=dash.meta.doc_type,
                                  id=dash.meta.id, body=dash.to_dict())

    def delete_kibana_objects(self):
        logger.info('deleting Kibana objects with prefix ' + self.prefix)

        search = elasticsearch_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=self.prefix)
        response = search.execute()

        for result in response:
            self.client.delete(index='.kibana', doc_type=result.meta.doc_type,
                               id=result.meta.id)

    def delete_elasticsearch_indices(self):
        logger.info(
            'deleting Elasticsearch indices with prefix ' + self.prefix)
        self.client.indices.delete(index=self.prefix + '_*')

    def index_task(self, task):
        logger.debug("parsing task object")

        task = dict([(m, o) for (m, o) in inspect.getmembers(task)
                     if not inspect.isroutine(o) and not m.startswith('__')])

        task.pop('_task')
        task.pop('command')

        task['resources_requested'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_requested'])
             if not inspect.isroutine(o) and not m.startswith('__')])
        task['resources_allocated'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_allocated'])
             if not inspect.isroutine(o) and not m.startswith('__')])
        task['resources_measured'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_measured'])
             if not inspect.isroutine(o) and not m.startswith('__')])

        task['resources_requested'].pop('this')
        task['resources_measured'].pop('this')
        task['resources_allocated'].pop('this')

        task['resources_measured'].pop('peak_times')

        task['send_input_start'] = datetime.utcfromtimestamp(
            float(str(task['send_input_start'])[:10]))
        task['send_input_finish'] = datetime.utcfromtimestamp(
            float(str(task['send_input_finish'])[:10]))
        task['execute_cmd_start'] = datetime.utcfromtimestamp(
            float(str(task['execute_cmd_start'])[:10]))
        task['execute_cmd_finish'] = datetime.utcfromtimestamp(
            float(str(task['execute_cmd_finish'])[:10]))
        task['receive_output_start'] = datetime.utcfromtimestamp(
            float(str(task['receive_output_start'])[:10]))
        task['receive_output_finish'] = datetime.utcfromtimestamp(
            float(str(task['receive_output_finish'])[:10]))
        task['submit_time'] = datetime.utcfromtimestamp(
            float(str(task['submit_time'])[:10]))
        task['finish_time'] = datetime.utcfromtimestamp(
            float(str(task['finish_time'])[:10]))

        task['resources_measured']['start'] = datetime.utcfromtimestamp(
            float(str(task['resources_measured']['start'])[:10]))
        task['resources_measured']['end'] = datetime.utcfromtimestamp(
            float(str(task['resources_measured']['end'])[:10]))

        task_log = task.pop('output')

        e_p = re.compile('Begin Fatal Exception([\s\S]*)End Fatal Exception')
        e_match = e_p.search(task_log)

        if e_match:
            logger.debug("parsing task.log fatal exception")

            task['fatal_exception']['message'] = e_match.group(1)

            e_cat_p = re.compile('\'(.*)\'')
            task['fatal_exception']['exception_category'] = \
                e_cat_p.search(task['fatal_exception']['message']).group(1)

            e_mess_p = re.compile('Exception Message:\n(.*)')
            task['fatal_exception']['exception_message'] = \
                e_mess_p.search(task['fatal_exception']['message']).group(1)

        upsert_doc = {'doc': {'task': task, 'timestamp': task['submit_time']},
                      'doc_as_upsert': True}

        try:
            self.client.update(index=self.prefix + '_lobster_tasks',
                               doc_type='task', id=task['id'],
                               body=upsert_doc)
            if self.populate_template:
                self.client.update(index='[template]_lobster_tasks',
                                   doc_type='task', id=task['id'],
                                   body=upsert_doc)
        except es.exceptions.ConnectionError as e:
            logger.error(e)

    def index_task_update(self, task_update):
        logger.debug("parsing task update")

        task_update = task_update.__dict__

        task_update['runtime'] = \
            task_update['time_processing_end'] - \
            task_update['time_wrapper_start']
        task_update['time_input_transfer'] = \
            task_update['time_transfer_in_start'] - \
            task_update['time_transfer_in_end']
        task_update['time_startup'] = \
            task_update['time_wrapper_start'] - \
            task_update['time_transfer_in_end']
        task_update['time_release_setup'] = \
            task_update['time_wrapper_ready'] - \
            task_update['time_wrapper_start']
        task_update['time_stage_in'] = \
            task_update['time_stage_in_end'] - \
            task_update['time_wrapper_ready']
        task_update['time_prologue'] = \
            task_update['time_prologue_end'] - \
            task_update['time_stage_in_end']
        task_update['time_overhead'] = \
            task_update['time_wrapper_ready'] - \
            task_update['time_wrapper_start']
        task_update['time_executable'] = \
            task_update['time_processing_end'] - \
            task_update['time_prologue_end']
        task_update['time_epilogue'] = \
            task_update['time_epilogue_end'] - \
            task_update['time_processing_end']
        task_update['time_stage_out'] = \
            task_update['time_stage_out_end'] - \
            task_update['time_epilogue_end']
        task_update['time_output_transfer_wait'] = \
            task_update['time_transfer_out_start'] - \
            task_update['time_stage_out_end']
        task_update['time_output_transfer_work_queue'] = \
            task_update['time_transfer_out_end'] - \
            task_update['time_transfer_out_start']

        task_update['time_processing_end'] = datetime.utcfromtimestamp(
            task_update['time_processing_end'])
        task_update['time_prologue_end'] = datetime.utcfromtimestamp(
            task_update['time_prologue_end'])
        task_update['time_retrieved'] = datetime.utcfromtimestamp(
            task_update['time_retrieved'])
        task_update['time_stage_in_end'] = datetime.utcfromtimestamp(
            task_update['time_stage_in_end'])
        task_update['time_stage_out_end'] = datetime.utcfromtimestamp(
            task_update['time_stage_out_end'])
        task_update['time_transfer_in_end'] = datetime.utcfromtimestamp(
            task_update['time_transfer_in_end'])
        task_update['time_transfer_in_start'] = datetime.utcfromtimestamp(
            task_update['time_transfer_in_start'])
        task_update['time_transfer_out_end'] = datetime.utcfromtimestamp(
            task_update['time_transfer_out_end'])
        task_update['time_transfer_out_start'] = datetime.utcfromtimestamp(
            task_update['time_transfer_out_start'])
        task_update['time_wrapper_ready'] = datetime.utcfromtimestamp(
            task_update['time_wrapper_ready'])
        task_update['time_wrapper_start'] = datetime.utcfromtimestamp(
            task_update['time_wrapper_start'])
        task_update['time_epilogue_end'] = datetime.utcfromtimestamp(
            task_update['time_epilogue_end'])

        upsert_doc = {'doc': {'task_update': task_update},
                      'doc_as_upsert': True}

        try:
            self.client.update(index=self.prefix + '_lobster_tasks',
                               doc_type='task', id=task_update['id'],
                               body=upsert_doc)
            if self.populate_template:
                self.client.update(index='[template]_lobster_tasks',
                                   doc_type='task', id=task_update['id'],
                                   body=upsert_doc)
        except es.exceptions.ConnectionError as e:
            logger.error(e)

    def index_work_queue(self, now, left, times, log_attributes, stats):
        logger.debug("parsing work queue log")

        keys = ["timestamp", "units_left"] + \
            ["total_{}_time".format(k) for k in sorted(times.keys())] + \
            log_attributes

        values = [datetime.utcfromtimestamp(int(now.strftime('%s'))), left] + \
            [times[k] for k in sorted(times.keys())] + \
            [getattr(stats, a) for a in log_attributes]

        wq = dict(zip(keys, values))

        try:
            self.client.index(index=self.prefix + '_work_queue',
                              doc_type='log', body=wq,
                              id=str(int(int(now.strftime('%s')) * 1e6 +
                                         now.microsecond)))
            if self.populate_template:
                self.client.index(index='[template]_work_queue',
                                  doc_type='log', body=wq,
                                  id=str(int(int(now.strftime('%s')) * 1e6 +
                                             now.microsecond)))
        except es.exceptions.ConnectionError as e:
            logger.error(e)
<commit_msg>Add category monitoring to lobster stats (formerly WQ).<commit_after>import elasticsearch as es
import elasticsearch_dsl as es_dsl
from datetime import datetime
import json
import re
import inspect
import logging

from lobster.util import Configurable

logger = logging.getLogger('lobster.monitor.elk')

class ElkInterface(Configurable):
    """
    Enables ELK stack monitoring for the current Lobster run using an existing
    Elasticsearch instance.

    Parameters
    ----------
        host : str
            Host running Elasticsearch cluster.
        port : int
            Port number running Elasticsearch HTTP service.
        user : str
            User ID to label Elasticsearch indices and Kibana objects.
        project : str
            Project ID to label Elasticsearch indices and Kibana objects.
        modules : list
            List of modules to include from the Kibana templates. Defaults to
            including only the core templates.
        populate_template : bool
            Whether to send documents to indices with the [template] prefix,
            in addition to sending them to the [user_run] prefix. Defaults to
            false.
    """
    _mutable = {}

    def __init__(self, host, port, user, project, modules=None,
                 populate_template=False):
        self.host = host
        self.port = port
        self.user = user
        self.project = project
        self.modules = modules or ['core']
        self.populate_template = populate_template
        self.prefix = '[' + self.user + '_' + self.project + ']'
        self.client = es.Elasticsearch([{'host': self.host,
                                         'port': self.port}])

    def __getstate__(self):
        state = {'host': self.host,
                 'port': self.port,
                 'user': self.user,
                 'project': self.project,
                 'modules': self.modules,
                 'prefix': self.prefix}
        return state

    def __setstate__(self, state):
        self.host = state['host']
        self.port = state['port']
        self.user = state['user']
        self.project = state['project']
        self.modules = state['modules']
        self.prefix = state['prefix']
        self.client = es.Elasticsearch([{'host': self.host,
                                         'port': self.port}])

    def check_prefix(self):
        logger.info("checking Elasticsearch and Kibana prefixes")

        try:
            indices = self.client.indices.get_aliases().keys()
        except es.exceptions.ConnectionError as e:
            logger.error(e)
            raise e

        if any(self.prefix in s for s in indices):
            e = AttributeError("Elasticsearch indices with prefix " +
                               self.prefix + " already exist.")
            logger.error(e)
            raise e

        search = es_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=self.prefix)
        response = search.execute()
        if len(response) > 0:
            e = AttributeError("Kibana objects with prefix " +
                               self.prefix + " already exist.")
            logger.error(e)
            raise e

    def generate_kibana_objects(self):
        logger.info("generating Kibana objects from templates")

        temp_prefix = '[template]'
        self.prefix = '[' + self.user + '_' + self.project + ']'
        other_prefix = re.compile('\[.*\]')

        search_index = es_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=temp_prefix) \
            .filter('match', _type='index-pattern')
        response_index = search_index.execute()

        for index in response_index:
            index.meta.id = index.meta.id.replace(temp_prefix, self.prefix, 1)
            index.title = index.title.replace(temp_prefix, self.prefix, 1)
            self.client.index(index='.kibana', doc_type=index.meta.doc_type,
                              id=index.meta.id, body=index.to_dict())

        for module in self.modules:
            temp_mod_prefix = temp_prefix + '[' + module + ']'
            new_mod_prefix = self.prefix + '[' + module + ']'

            search_vis = es_dsl.Search(using=self.client, index='.kibana') \
                .filter('prefix', _id=temp_mod_prefix) \
                .filter('match', _type='visualization')

            search_dash = es_dsl.Search(using=self.client, index='.kibana') \
                .filter('prefix', _id=temp_mod_prefix) \
                .filter('match', _type='dashboard')

            response_vis = search_vis.execute()
            response_dash = search_dash.execute()

            for vis in response_vis:
                vis.meta.id = vis.meta.id.replace(
                    temp_mod_prefix, new_mod_prefix, 1)
                vis.title = vis.title.replace(
                    temp_mod_prefix, new_mod_prefix, 1)

                source = json.loads(vis.kibanaSavedObjectMeta.searchSourceJSON)
                source['index'] = other_prefix.sub(
                    self.prefix, source['index'])
                vis.kibanaSavedObjectMeta.searchSourceJSON = json.dumps(source)

                self.client.index(index='.kibana', doc_type=vis.meta.doc_type,
                                  id=vis.meta.id, body=vis.to_dict())

            for dash in response_dash:
                dash.meta.id = dash.meta.id.replace(
                    temp_mod_prefix, new_mod_prefix, 1)
                dash.title = dash.title.replace(
                    temp_mod_prefix, new_mod_prefix, 1)

                dash_panels = json.loads(dash.panelsJSON)
                for panel in dash_panels:
                    panel['id'] = panel['id'].replace(
                        temp_mod_prefix, new_mod_prefix, 1)
                dash.panelsJSON = json.dumps(dash_panels)

                self.client.index(index='.kibana', doc_type=dash.meta.doc_type,
                                  id=dash.meta.id, body=dash.to_dict())

    def delete_kibana_objects(self):
        logger.info('deleting Kibana objects with prefix ' + self.prefix)

        search = elasticsearch_dsl.Search(using=self.client, index='.kibana') \
            .filter('prefix', _id=self.prefix)
        response = search.execute()

        for result in response:
            self.client.delete(index='.kibana', doc_type=result.meta.doc_type,
                               id=result.meta.id)

    def delete_elasticsearch_indices(self):
        logger.info(
            'deleting Elasticsearch indices with prefix ' + self.prefix)
        self.client.indices.delete(index=self.prefix + '_*')

    def index_task(self, task):
        logger.debug("parsing task object")

        task = dict([(m, o) for (m, o) in inspect.getmembers(task)
                     if not inspect.isroutine(o) and not m.startswith('__')])

        task.pop('_task')
        task.pop('command')

        task['resources_requested'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_requested'])
             if not inspect.isroutine(o) and not m.startswith('__')])
        task['resources_allocated'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_allocated'])
             if not inspect.isroutine(o) and not m.startswith('__')])
        task['resources_measured'] = dict(
            [(m, o) for (m, o) in
             inspect.getmembers(task['resources_measured'])
             if not inspect.isroutine(o) and not m.startswith('__')])

        task['resources_requested'].pop('this')
        task['resources_measured'].pop('this')
        task['resources_allocated'].pop('this')

        task['resources_measured'].pop('peak_times')

        task['send_input_start'] = datetime.utcfromtimestamp(
            float(str(task['send_input_start'])[:10]))
        task['send_input_finish'] = datetime.utcfromtimestamp(
            float(str(task['send_input_finish'])[:10]))
        task['execute_cmd_start'] = datetime.utcfromtimestamp(
            float(str(task['execute_cmd_start'])[:10]))
        task['execute_cmd_finish'] = datetime.utcfromtimestamp(
            float(str(task['execute_cmd_finish'])[:10]))
        task['receive_output_start'] = datetime.utcfromtimestamp(
            float(str(task['receive_output_start'])[:10]))
        task['receive_output_finish'] = datetime.utcfromtimestamp(
            float(str(task['receive_output_finish'])[:10]))
        task['submit_time'] = datetime.utcfromtimestamp(
            float(str(task['submit_time'])[:10]))
        task['finish_time'] = datetime.utcfromtimestamp(
            float(str(task['finish_time'])[:10]))

        task['resources_measured']['start'] = datetime.utcfromtimestamp(
            float(str(task['resources_measured']['start'])[:10]))
        task['resources_measured']['end'] = datetime.utcfromtimestamp(
            float(str(task['resources_measured']['end'])[:10]))

        task_log = task.pop('output')

        e_p = re.compile('Begin Fatal Exception([\s\S]*)End Fatal Exception')
        e_match = e_p.search(task_log)

        if e_match:
            logger.debug("parsing task.log fatal exception")

            task['fatal_exception']['message'] = e_match.group(1)

            e_cat_p = re.compile('\'(.*)\'')
            task['fatal_exception']['exception_category'] = \
                e_cat_p.search(task['fatal_exception']['message']).group(1)

            e_mess_p = re.compile('Exception Message:\n(.*)')
            task['fatal_exception']['exception_message'] = \
                e_mess_p.search(task['fatal_exception']['message']).group(1)

        upsert_doc = {'doc': {'task': task, 'timestamp': task['submit_time']},
                      'doc_as_upsert': True}

        try:
            self.client.update(index=self.prefix + '_lobster_tasks',
                               doc_type='task', id=task['id'],
                               body=upsert_doc)
            if self.populate_template:
                self.client.update(index='[template]_lobster_tasks',
                                   doc_type='task', id=task['id'],
                                   body=upsert_doc)
        except es.exceptions.ConnectionError as e:
            logger.error(e)

    def index_task_update(self, task_update):
        logger.debug("parsing task update")

        task_update = task_update.__dict__

        task_update['runtime'] = \
            task_update['time_processing_end'] - \
            task_update['time_wrapper_start']
        task_update['time_input_transfer'] = \
            task_update['time_transfer_in_start'] - \
            task_update['time_transfer_in_end']
        task_update['time_startup'] = \
            task_update['time_wrapper_start'] - \
            task_update['time_transfer_in_end']
        task_update['time_release_setup'] = \
            task_update['time_wrapper_ready'] - \
            task_update['time_wrapper_start']
        task_update['time_stage_in'] = \
            task_update['time_stage_in_end'] - \
            task_update['time_wrapper_ready']
        task_update['time_prologue'] = \
            task_update['time_prologue_end'] - \
            task_update['time_stage_in_end']
        task_update['time_overhead'] = \
            task_update['time_wrapper_ready'] - \
            task_update['time_wrapper_start']
        task_update['time_executable'] = \
            task_update['time_processing_end'] - \
            task_update['time_prologue_end']
        task_update['time_epilogue'] = \
            task_update['time_epilogue_end'] - \
            task_update['time_processing_end']
        task_update['time_stage_out'] = \
            task_update['time_stage_out_end'] - \
            task_update['time_epilogue_end']
        task_update['time_output_transfer_wait'] = \
            task_update['time_transfer_out_start'] - \
            task_update['time_stage_out_end']
        task_update['time_output_transfer_work_queue'] = \
            task_update['time_transfer_out_end'] - \
            task_update['time_transfer_out_start']

        task_update['time_processing_end'] = datetime.utcfromtimestamp(
            task_update['time_processing_end'])
        task_update['time_prologue_end'] = datetime.utcfromtimestamp(
            task_update['time_prologue_end'])
        task_update['time_retrieved'] = datetime.utcfromtimestamp(
            task_update['time_retrieved'])
        task_update['time_stage_in_end'] = datetime.utcfromtimestamp(
            task_update['time_stage_in_end'])
        task_update['time_stage_out_end'] = datetime.utcfromtimestamp(
            task_update['time_stage_out_end'])
        task_update['time_transfer_in_end'] = datetime.utcfromtimestamp(
            task_update['time_transfer_in_end'])
        task_update['time_transfer_in_start'] = datetime.utcfromtimestamp(
            task_update['time_transfer_in_start'])
        task_update['time_transfer_out_end'] = datetime.utcfromtimestamp(
            task_update['time_transfer_out_end'])
        task_update['time_transfer_out_start'] = datetime.utcfromtimestamp(
            task_update['time_transfer_out_start'])
        task_update['time_wrapper_ready'] = datetime.utcfromtimestamp(
            task_update['time_wrapper_ready'])
        task_update['time_wrapper_start'] = datetime.utcfromtimestamp(
            task_update['time_wrapper_start'])
        task_update['time_epilogue_end'] = datetime.utcfromtimestamp(
            task_update['time_epilogue_end'])

        upsert_doc = {'doc': {'task_update': task_update},
                      'doc_as_upsert': True}

        try:
            self.client.update(index=self.prefix + '_lobster_tasks',
                               doc_type='task', id=task_update['id'],
                               body=upsert_doc)
            if self.populate_template:
                self.client.update(index='[template]_lobster_tasks',
                                   doc_type='task', id=task_update['id'],
                                   body=upsert_doc)
        except es.exceptions.ConnectionError as e:
            logger.error(e)

    def index_stats(self, now, left, times, log_attributes, stats, category):
        logger.debug("parsing lobster stats log")

        keys = ['timestamp', 'units_left'] + \
            ['total_{}_time'.format(k) for k in sorted(times.keys())] + \
            log_attributes + ['category']

        values = [datetime.utcfromtimestamp(int(now.strftime('%s'))), left] + \
            [times[k] for k in sorted(times.keys())] + \
            [getattr(stats, a) for a in log_attributes] + [category]

        stats = dict(zip(keys, values))

        try:
            self.client.index(index=self.prefix + '_lobster_stats',
                              doc_type='log', body=stats,
                              id=str(int(int(now.strftime('%s')) * 1e6 +
                                         now.microsecond)))
            if self.populate_template:
                self.client.index(index='[template]_lobster_stats',
                                  doc_type='log', body=stats,
                                  id=str(int(int(now.strftime('%s')) * 1e6 +
                                             now.microsecond)))
        except es.exceptions.ConnectionError as e:
            logger.error(e)
<|endoftext|>


<commit_before>from scripts import settings, streaming

# Query keys categorize tweets
# Each key or category corresponds to an array of keywords
queries = {'ETH': ['ETH', 'Ethereum'],
           'LTC': ['LTC', 'Litecoin'],
           'BTC': ['BTC', 'Bitcoin'],
           'XRP': ['XRP', 'Ripple'],
           'XLM': ['XLM', 'Stellar']}

# Aggregate volume and sentiment every 15 minutes
refresh = 15*60

streaming.stream(settings.credentials,
                 queries,
                 refresh,
                 sentiment=True,
                 debug=True)
<commit_msg>Update listen.py<commit_after>from scripts import settings, streaming

# Query keys categorize tweets
# Each key or category corresponds to an array of keywords
queries = {'ETH': ['ETH', 'Ethereum'],
           'LTC': ['LTC', 'Litecoin'],
           'BTC': ['BTC', 'Bitcoin'],
           'XRP': ['XRP', 'Ripple'],
           'XLM': ['XLM', 'Stellar']}

# Aggregate volume and sentiment every 15 minutes
refresh = 15*60

streaming.streamer(settings.credentials,
                   queries,
                   refresh,
                   sentiment=True,
                   debug=True)
<|endoftext|>


<commit_before>import logging
from collections import defaultdict

from billy import db
from billy.conf import settings
from billy.utils import term_for_session
from billy.reports.utils import get_quality_exceptions, combine_reports

logger = logging.getLogger('billy')

def _vote_report_dict():
    return {'vote_count': 0,
            '_passed_vote_count': 0,
            'votes_per_month': defaultdict(int),
            'votes_per_chamber': defaultdict(int),
            'votes_per_type': defaultdict(int),
            'bad_vote_counts': set(),
            '_rollcall_count': 0,
            '_rollcalls_with_leg_id_count': 0,
            'unmatched_voters': set(),
           }

def scan_votes(abbr):
    sessions = defaultdict(_vote_report_dict)

    # load exception data into sets of ids indexed by exception type
    quality_exceptions = get_quality_exceptions(abbr)

    for vote in db.votes.find({settings.LEVEL_FIELD: abbr}):
        session_d = sessions[vote['session']]

        session_d['vote_count'] += 1
        if vote['passed']:
            session_d['_passed_vote_count'] += 1
        session_d['votes_per_chamber'][vote['chamber']] += 1
        if not vote.get('type'):
            logger.warning('vote %s missing type' % vote['_id'])
            continue
        session_d['votes_per_type'][vote.get('type')] += 1
        if not vote.get('date'):
            logger.warning('vote %s missing date' % vote['_id'])
            continue
        session_d['votes_per_month'][vote['date'].strftime('%Y-%m')] += 1

        # roll calls
        has_rollcalls = False
        for rc in (vote['yes_votes'] + vote['no_votes'] +
                   vote['other_votes']):
            has_rollcalls = True
            session_d['_rollcall_count'] += 1
            if rc.get('leg_id'):
                session_d['_rollcalls_with_leg_id_count'] += 1
            else:
                # keep missing leg_ids
                session_d['unmatched_voters'].add(
                    (term_for_session(abbr, vote['session']),
                     vote['chamber'],
                    rc['name'])
                )

        # check counts if any rollcalls are present
        if has_rollcalls:
            if (len(vote['yes_votes']) != vote['yes_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_yes_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])
            if (len(vote['no_votes']) != vote['no_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_no_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])
            if (len(vote['other_votes']) != vote['other_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_other_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])

    # do logging of unnecessary exceptions
    for qe_type, qes in quality_exceptions.iteritems():
        if qes:
            logger.warning('unnecessary {0} exceptions for {1} votes: \n  {2}'
                           .format(qe_type, len(qes), '\n  '.join(qes)))

    return {'sessions': sessions}

def calculate_percentages(report):
    vote_count = float(report['vote_count']) / 100
    if vote_count:
        report['votes_passed'] = report.pop('_passed_vote_count') / vote_count
        for k in report['votes_per_type'].iterkeys():
            report['votes_per_type'][k] /= vote_count
        for k in report['votes_per_chamber'].iterkeys():
            report['votes_per_chamber'][k] /= vote_count
        for k in report['votes_per_month'].iterkeys():
            report['votes_per_month'][k] /= vote_count
    rollcall_count = float(report.pop('_rollcall_count')) / 100
    if rollcall_count:
        report['rollcalls_with_leg_id'] = (
            report.pop('_rollcalls_with_leg_id_count') / rollcall_count
        )

def vote_report(abbr):
    report = scan_votes(abbr)
    combined_report = combine_reports(report['sessions'], _vote_report_dict())
    for session in report['sessions'].itervalues():
        calculate_percentages(session)
    calculate_percentages(combined_report)
    report.update(combined_report)
    return report
<commit_msg>from billy.core import db<commit_after>import logging
from collections import defaultdict

from billy.core import db
from billy.conf import settings
from billy.utils import term_for_session
from billy.reports.utils import get_quality_exceptions, combine_reports

logger = logging.getLogger('billy')

def _vote_report_dict():
    return {'vote_count': 0,
            '_passed_vote_count': 0,
            'votes_per_month': defaultdict(int),
            'votes_per_chamber': defaultdict(int),
            'votes_per_type': defaultdict(int),
            'bad_vote_counts': set(),
            '_rollcall_count': 0,
            '_rollcalls_with_leg_id_count': 0,
            'unmatched_voters': set(),
           }

def scan_votes(abbr):
    sessions = defaultdict(_vote_report_dict)

    # load exception data into sets of ids indexed by exception type
    quality_exceptions = get_quality_exceptions(abbr)

    for vote in db.votes.find({settings.LEVEL_FIELD: abbr}):
        session_d = sessions[vote['session']]

        session_d['vote_count'] += 1
        if vote['passed']:
            session_d['_passed_vote_count'] += 1
        session_d['votes_per_chamber'][vote['chamber']] += 1
        if not vote.get('type'):
            logger.warning('vote %s missing type' % vote['_id'])
            continue
        session_d['votes_per_type'][vote.get('type')] += 1
        if not vote.get('date'):
            logger.warning('vote %s missing date' % vote['_id'])
            continue
        session_d['votes_per_month'][vote['date'].strftime('%Y-%m')] += 1

        # roll calls
        has_rollcalls = False
        for rc in (vote['yes_votes'] + vote['no_votes'] +
                   vote['other_votes']):
            has_rollcalls = True
            session_d['_rollcall_count'] += 1
            if rc.get('leg_id'):
                session_d['_rollcalls_with_leg_id_count'] += 1
            else:
                # keep missing leg_ids
                session_d['unmatched_voters'].add(
                    (term_for_session(abbr, vote['session']),
                     vote['chamber'],
                    rc['name'])
                )

        # check counts if any rollcalls are present
        if has_rollcalls:
            if (len(vote['yes_votes']) != vote['yes_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_yes_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])
            if (len(vote['no_votes']) != vote['no_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_no_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])
            if (len(vote['other_votes']) != vote['other_count'] and
                vote['vote_id'] not in
                quality_exceptions['votes:bad_other_count']):
                session_d['bad_vote_counts'].add(vote['vote_id'])

    # do logging of unnecessary exceptions
    for qe_type, qes in quality_exceptions.iteritems():
        if qes:
            logger.warning('unnecessary {0} exceptions for {1} votes: \n  {2}'
                           .format(qe_type, len(qes), '\n  '.join(qes)))

    return {'sessions': sessions}

def calculate_percentages(report):
    vote_count = float(report['vote_count']) / 100
    if vote_count:
        report['votes_passed'] = report.pop('_passed_vote_count') / vote_count
        for k in report['votes_per_type'].iterkeys():
            report['votes_per_type'][k] /= vote_count
        for k in report['votes_per_chamber'].iterkeys():
            report['votes_per_chamber'][k] /= vote_count
        for k in report['votes_per_month'].iterkeys():
            report['votes_per_month'][k] /= vote_count
    rollcall_count = float(report.pop('_rollcall_count')) / 100
    if rollcall_count:
        report['rollcalls_with_leg_id'] = (
            report.pop('_rollcalls_with_leg_id_count') / rollcall_count
        )

def vote_report(abbr):
    report = scan_votes(abbr)
    combined_report = combine_reports(report['sessions'], _vote_report_dict())
    for session in report['sessions'].itervalues():
        calculate_percentages(session)
    calculate_percentages(combined_report)
    report.update(combined_report)
    return report
<|endoftext|>


<commit_before>import re

RegexType = type(re.compile(""))

class Profile(object):
    __profiles__ = {}

    @classmethod
    def get_profile(cls, name):
        return cls.__profiles__.get(name, None)

    def __init__(self, name=None, tags=None, priorities=None, filters=None,
            buffers=None, wrap=True, device=None, emulator=None, format=None,
            packages=None):
        if not name:
            raise Exception("Profile is missing a name")

        self.name = name
        self.__profiles__[name] = self

        self.init_tags(tags)
        self.init_priorities(priorities)
        self.init_filters(filters)
        self.init_packages(packages)
        self.buffers = buffers
        self.wrap = wrap
        self.device = device
        self.emulator = emulator
        self.format = format

    def init_packages(self, packages):

        self.pid_map = {}
        self.package_search = {}

        if packages:
            for package in packages:
                search_string = 'Start proc ' + package
                regex = re.compile(search_string + ".*?pid=(\\d+)", re.IGNORECASE|re.DOTALL)
                self.package_search[package] = (search_string, regex)

    def init_tags(self, tags):
        self.tags = None
        self.tag_colors = None
        if isinstance(tags, dict):
            self.tags = tags.keys()
            self.tag_colors = tags
        elif isinstance(tags, (list, tuple)):
            self.tags = tags
        elif tags:
            self.tags = (tags)

    def init_priorities(self, priorities):
        self.priorities = None
        if isinstance(priorities, (list, tuple)):
            self.priorities = priorities
        elif priorities:
            self.priorities = (priorities)

    def init_filters(self, filters):
        self.filters = []
        if not filters:
            return

        if not isinstance(filters, (list, tuple)):
            filters = [filters]

        for filter in filters:
            if isinstance(filter, (str, RegexType)):
                self.filters.append(self.regex_filter(filter))
            else:
                self.filters.append(filter)

    def regex_filter(self, regex):
        pattern = regex
        if not isinstance(regex, RegexType):
            pattern = re.compile(regex)

        def __filter(data):
            if "message" not in data:
                return True
            return pattern.search(data["message"])
        return __filter

    def process_new_pid(self, data):
        string = data.get('message')
        if string and string.startswith('Start proc'):
            for package in self.package_search:
                match = self.package_search[package][1].search(string)
                if match:
                    self.pid_map[package] = match.group(1)

    def include(self, data):
        if not data:
            raise Exception("data should not be None")

        self.process_new_pid(data)  #process pid

        if self.tags and data.get("tag") not in self.tags:
            return False

        if self.priorities and data.get("priority") not in self.priorities:
            return False

        if self.package_search and data.get("pid") not in self.pid_map.values():
            return False

        if not self.filters:
            return True

        for filter in self.filters:
            if not filter(data):
                return False

        return True
<commit_msg>testing both regular and 5.1 log line formats<commit_after>import re

RegexType = type(re.compile(""))

class Profile(object):
    __profiles__ = {}

    @classmethod
    def get_profile(cls, name):
        return cls.__profiles__.get(name, None)

    def __init__(self, name=None, tags=None, priorities=None, filters=None,
            buffers=None, wrap=True, device=None, emulator=None, format=None,
            packages=None):
        if not name:
            raise Exception("Profile is missing a name")

        self.name = name
        self.__profiles__[name] = self

        self.init_tags(tags)
        self.init_priorities(priorities)
        self.init_filters(filters)
        self.init_packages(packages)
        self.buffers = buffers
        self.wrap = wrap
        self.device = device
        self.emulator = emulator
        self.format = format

    def init_packages(self, packages):

        self.pid_map = {}
        self.package_search = {}

        if packages:
            for package in packages:
                search_string = 'Start proc ' + package
                regex = re.compile(search_string + ".*?pid=(\\d+)", re.IGNORECASE|re.DOTALL)
                regex51 = re.compile(r'Start proc (\d+):' + package)

                self.package_search[package] = (search_string, regex, regex51)

    def init_tags(self, tags):
        self.tags = None
        self.tag_colors = None
        if isinstance(tags, dict):
            self.tags = tags.keys()
            self.tag_colors = tags
        elif isinstance(tags, (list, tuple)):
            self.tags = tags
        elif tags:
            self.tags = (tags)

    def init_priorities(self, priorities):
        self.priorities = None
        if isinstance(priorities, (list, tuple)):
            self.priorities = priorities
        elif priorities:
            self.priorities = (priorities)

    def init_filters(self, filters):
        self.filters = []
        if not filters:
            return

        if not isinstance(filters, (list, tuple)):
            filters = [filters]

        for filter in filters:
            if isinstance(filter, (str, RegexType)):
                self.filters.append(self.regex_filter(filter))
            else:
                self.filters.append(filter)

    def regex_filter(self, regex):
        pattern = regex
        if not isinstance(regex, RegexType):
            pattern = re.compile(regex)

        def __filter(data):
            if "message" not in data:
                return True
            return pattern.search(data["message"])
        return __filter

    def process_new_pid(self, data):
        string = data.get('message')
        if string and string.startswith('Start proc'):
            for package in self.package_search:
                match = self.package_search[package][1].search(string)
                if match:
                    self.pid_map[package] = match.group(1)
                match51 = self.package_search[package][2].search(string)
                if match51:
                    self.pid_map[package] = match51.group(1)

    def include(self, data):
        if not data:
            raise Exception("data should not be None")

        self.process_new_pid(data)  #process pid

        if self.tags and data.get("tag") not in self.tags:
            return False

        if self.priorities and data.get("priority") not in self.priorities:
            return False

        if self.package_search and data.get("pid") not in self.pid_map.values():
            return False

        if not self.filters:
            return True

        for filter in self.filters:
            if not filter(data):
                return False

        return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Sep 01 20:22:25 2015

@author: matsumi
"""

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits

def sigmoid(s):
    y = 1 / (1 + np.exp(-s))
    return y

# main文
if __name__ == '__main__':
    digits = load_digits(2)
    images = digits.images
    plt.matshow(images[4], cmap=plt.cm.gray)
    plt.show()

    # データ・セットの読み込み
    X_raw = digits.data / 16.0
    t = digits.target
    num_examples = len(X_raw)
    x = X_raw[0]
    X = np.hstack((X_raw, np.ones((num_examples, 1))))

    # ρを定義する(ρ=0.1で良いか判断し，収束しなければ値を変える．)
    rho = 1.4

    # wを定義する（65個の個数を持った配列をrandomで作成する）
    w = np.random.randn(65)

    # 収束するまで繰り返す
    max_iteration = 100

    # 確率的勾配降下法
    for epoch in range(max_iteration):
        for x_i, t_i in zip(X, t):
            y_i = sigmoid(np.inner(w, x_i))
            w_new = w - rho * (y_i - t_i) * x_i
            w = w_new

        # 負の対数尤度関数の値を表示する
        y = sigmoid(np.inner(w, X))
        error = np.sum(-(t*(np.log(y)) + (1 - t)*np.log(1 - y)))
        print "error:", error

        # 正解クラスと予測クラスとの比較
        predict_class = y >= 0.5
        num_correct = np.sum(t == predict_class)
        correct_percent = num_correct / float(num_examples) * 100
        print "correct_percent:", correct_percent
        if correct_percent == 100.0:
            break

    # 予測クラスと真のクラスを表示する
    print "predict_class:", predict_class
    print "t:", t == 1

    # wの可視化
    w_true = w[0:64]  # w_trueとは結果をプロットするために定義したものである
    print "w_true:", w_true
    plt.matshow(w_true.reshape(8, 8), cmap=plt.cm.gray)
    plt.show()
<commit_msg>誤差がinf又はnanになっていないか確認する<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Sep 01 20:22:25 2015

@author: matsumi
"""

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits

def sigmoid(s):
    y = 1 / (1 + np.exp(-s))
    return y

# main文
if __name__ == '__main__':
    digits = load_digits(2)
    images = digits.images
    plt.matshow(images[4], cmap=plt.cm.gray)
    plt.show()

    # データ・セットの読み込み
    X_raw = digits.data / 16.0
    t = digits.target
    num_examples = len(X_raw)
    x = X_raw[0]
    X = np.hstack((X_raw, np.ones((num_examples, 1))))

    # ρを定義する(ρ=0.1で良いか判断し，収束しなければ値を変える．)
    rho = 1.4

    # wを定義する（65個の個数を持った配列をrandomで作成する）
    w = np.random.randn(65)

    # 収束するまで繰り返す
    max_iteration = 100

    # 確率的勾配降下法
    for epoch in range(max_iteration):
        for x_i, t_i in zip(X, t):
            y_i = sigmoid(np.inner(w, x_i))
            w_new = w - rho * (y_i - t_i) * x_i
            w = w_new

        # 負の対数尤度関数の値を表示する
        y = sigmoid(np.inner(w, X))
        error = np.sum(-(t*(np.log(y)) + (1 - t)*np.log(1 - y)))
        assert not np.any(np.isnan(error))
        assert not np.any(np.isinf(error))
        print "error:", error

        # 正解クラスと予測クラスとの比較
        predict_class = y >= 0.5
        num_correct = np.sum(t == predict_class)
        correct_percent = num_correct / float(num_examples) * 100
        print "correct_percent:", correct_percent
        if correct_percent == 100.0:
            break

    # 予測クラスと真のクラスを表示する
    print "predict_class:", predict_class
    print "t:", t == 1

    # wの可視化
    w_true = w[0:64]  # w_trueとは結果をプロットするために定義したものである
    print "w_true:", w_true
    plt.matshow(w_true.reshape(8, 8), cmap=plt.cm.gray)
    plt.show()
<|endoftext|>