<commit_before>"""distutils.command.check

Implements the Distutils 'check' command.
"""
from distutils.core import Command
from distutils.errors import DistutilsSetupError

try:
    # docutils is installed
    from docutils.utils import Reporter
    from docutils.parsers.rst import Parser
    from docutils import frontend
    from docutils import nodes
    from io import StringIO

    class SilentReporter(Reporter):

        def __init__(self, source, report_level, halt_level, stream=None,
                     debug=0, encoding='ascii', error_handler='replace'):
            self.messages = []
            Reporter.__init__(self, source, report_level, halt_level, stream,
                              debug, encoding, error_handler)

        def system_message(self, level, message, *children, **kwargs):
            self.messages.append((level, message, children, kwargs))
            return nodes.system_message(message, level=level,
                                        type=self.levels[level],
                                        *children, **kwargs)

    HAS_DOCUTILS = True
except Exception:
    # Catch all exceptions because exceptions besides ImportError probably
    # indicate that docutils is not ported to Py3k.
    HAS_DOCUTILS = False

class check(Command):
    """This command checks the meta-data of the package.
    """
    description = ("perform some checks on the package")
    user_options = [('metadata', 'm', 'Verify meta-data'),
                    ('restructuredtext', 'r',
                     ('Checks if long string meta-data syntax '
                      'are reStructuredText-compliant')),
                    ('strict', 's',
                     'Will exit with an error if a check fails')]

    boolean_options = ['metadata', 'restructuredtext', 'strict']

    def initialize_options(self):
        """Sets default values for options."""
        self.restructuredtext = 0
        self.metadata = 1
        self.strict = 0
        self._warnings = 0

    def finalize_options(self):
        pass

    def warn(self, msg):
        """Counts the number of warnings that occurs."""
        self._warnings += 1
        return Command.warn(self, msg)

    def run(self):
        """Runs the command."""
        # perform the various tests
        if self.metadata:
            self.check_metadata()
        if self.restructuredtext:
            if HAS_DOCUTILS:
                self.check_restructuredtext()
            elif self.strict:
                raise DistutilsSetupError('The docutils package is needed.')

        # let's raise an error in strict mode, if we have at least
        # one warning
        if self.strict and self._warnings > 0:
            raise DistutilsSetupError('Please correct your package.')

    def check_metadata(self):
        """Ensures that all required elements of meta-data are supplied.

        name, version, URL, (author and author_email) or
        (maintainer and maintainer_email)).

        Warns if any are missing.
        """
        metadata = self.distribution.metadata

        missing = []
        for attr in ('name', 'version', 'url'):
            if not (hasattr(metadata, attr) and getattr(metadata, attr)):
                missing.append(attr)

        if missing:
            self.warn("missing required meta-data: %s"  % ', '.join(missing))
        if metadata.author:
            if not metadata.author_email:
                self.warn("missing meta-data: if 'author' supplied, " +
                          "'author_email' must be supplied too")
        elif metadata.maintainer:
            if not metadata.maintainer_email:
                self.warn("missing meta-data: if 'maintainer' supplied, " +
                          "'maintainer_email' must be supplied too")
        else:
            self.warn("missing meta-data: either (author and author_email) " +
                      "or (maintainer and maintainer_email) " +
                      "must be supplied")

    def check_restructuredtext(self):
        """Checks if the long string fields are reST-compliant."""
        data = self.distribution.get_long_description()
        for warning in self._check_rst_data(data):
            line = warning[-1].get('line')
            if line is None:
                warning = warning[1]
            else:
                warning = '%s (line %s)' % (warning[1], line)
            self.warn(warning)

    def _check_rst_data(self, data):
        """Returns warnings when the provided data doesn't compile."""
        # the include and csv_table directives need this to be a path
        source_path = self.distribution.script_name or 'setup.py'
        parser = Parser()
        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
        settings.tab_width = 4
        settings.pep_references = None
        settings.rfc_references = None
        reporter = SilentReporter(source_path,
                          settings.report_level,
                          settings.halt_level,
                          stream=settings.warning_stream,
                          debug=settings.debug,
                          encoding=settings.error_encoding,
                          error_handler=settings.error_encoding_error_handler)

        document = nodes.document(settings, reporter, source=source_path)
        document.note_source(source_path, -1)
        try:
            parser.parse(data, document)
        except AttributeError as e:
            reporter.messages.append(
                (-1, 'Could not finish the parsing: %s.' % e, '', {}))

        return reporter.messages
<commit_msg>bpo-38914 Do not require email field in setup.py. (GH-17388)<commit_after>"""distutils.command.check

Implements the Distutils 'check' command.
"""
from distutils.core import Command
from distutils.errors import DistutilsSetupError

try:
    # docutils is installed
    from docutils.utils import Reporter
    from docutils.parsers.rst import Parser
    from docutils import frontend
    from docutils import nodes
    from io import StringIO

    class SilentReporter(Reporter):

        def __init__(self, source, report_level, halt_level, stream=None,
                     debug=0, encoding='ascii', error_handler='replace'):
            self.messages = []
            Reporter.__init__(self, source, report_level, halt_level, stream,
                              debug, encoding, error_handler)

        def system_message(self, level, message, *children, **kwargs):
            self.messages.append((level, message, children, kwargs))
            return nodes.system_message(message, level=level,
                                        type=self.levels[level],
                                        *children, **kwargs)

    HAS_DOCUTILS = True
except Exception:
    # Catch all exceptions because exceptions besides ImportError probably
    # indicate that docutils is not ported to Py3k.
    HAS_DOCUTILS = False

class check(Command):
    """This command checks the meta-data of the package.
    """
    description = ("perform some checks on the package")
    user_options = [('metadata', 'm', 'Verify meta-data'),
                    ('restructuredtext', 'r',
                     ('Checks if long string meta-data syntax '
                      'are reStructuredText-compliant')),
                    ('strict', 's',
                     'Will exit with an error if a check fails')]

    boolean_options = ['metadata', 'restructuredtext', 'strict']

    def initialize_options(self):
        """Sets default values for options."""
        self.restructuredtext = 0
        self.metadata = 1
        self.strict = 0
        self._warnings = 0

    def finalize_options(self):
        pass

    def warn(self, msg):
        """Counts the number of warnings that occurs."""
        self._warnings += 1
        return Command.warn(self, msg)

    def run(self):
        """Runs the command."""
        # perform the various tests
        if self.metadata:
            self.check_metadata()
        if self.restructuredtext:
            if HAS_DOCUTILS:
                self.check_restructuredtext()
            elif self.strict:
                raise DistutilsSetupError('The docutils package is needed.')

        # let's raise an error in strict mode, if we have at least
        # one warning
        if self.strict and self._warnings > 0:
            raise DistutilsSetupError('Please correct your package.')

    def check_metadata(self):
        """Ensures that all required elements of meta-data are supplied.

        Required fields:
            name, version, URL

        Recommended fields:
            (author and author_email) or (maintainer and maintainer_email))

        Warns if any are missing.
        """
        metadata = self.distribution.metadata

        missing = []
        for attr in ('name', 'version', 'url'):
            if not (hasattr(metadata, attr) and getattr(metadata, attr)):
                missing.append(attr)

        if missing:
            self.warn("missing required meta-data: %s"  % ', '.join(missing))
        if metadata.author:
            if not metadata.author_email:
                self.warn("missing meta-data: if 'author' supplied, " +
                          "'author_email' should be supplied too")
        elif metadata.maintainer:
            if not metadata.maintainer_email:
                self.warn("missing meta-data: if 'maintainer' supplied, " +
                          "'maintainer_email' should be supplied too")
        else:
            self.warn("missing meta-data: either (author and author_email) " +
                      "or (maintainer and maintainer_email) " +
                      "should be supplied")

    def check_restructuredtext(self):
        """Checks if the long string fields are reST-compliant."""
        data = self.distribution.get_long_description()
        for warning in self._check_rst_data(data):
            line = warning[-1].get('line')
            if line is None:
                warning = warning[1]
            else:
                warning = '%s (line %s)' % (warning[1], line)
            self.warn(warning)

    def _check_rst_data(self, data):
        """Returns warnings when the provided data doesn't compile."""
        # the include and csv_table directives need this to be a path
        source_path = self.distribution.script_name or 'setup.py'
        parser = Parser()
        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
        settings.tab_width = 4
        settings.pep_references = None
        settings.rfc_references = None
        reporter = SilentReporter(source_path,
                          settings.report_level,
                          settings.halt_level,
                          stream=settings.warning_stream,
                          debug=settings.debug,
                          encoding=settings.error_encoding,
                          error_handler=settings.error_encoding_error_handler)

        document = nodes.document(settings, reporter, source=source_path)
        document.note_source(source_path, -1)
        try:
            parser.parse(data, document)
        except AttributeError as e:
            reporter.messages.append(
                (-1, 'Could not finish the parsing: %s.' % e, '', {}))

        return reporter.messages
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2016 Intel Corporation.
#
# SPDX-License-Identifier: Apache-2.0

import copy
import logging
import os
import re
import sys
import threading

try:
    import ply.lex as lex
    import ply.yacc as yacc
except ImportError:
    sys.exit("PLY library for Python 3 not installed.\n"
             "Please install the ply package using your workstation's\n"
             "package manager or the 'pip' tool.")

_logger = logging.getLogger('twister')

reserved = {
    'and' : 'AND',
    'or' : 'OR',
    'not' : 'NOT',
    'in' : 'IN',
}

tokens = [
    "HEX",
    "STR",
    "INTEGER",
    "EQUALS",
    "NOTEQUALS",
    "LT",
    "GT",
    "LTEQ",
    "GTEQ",
    "OPAREN",
    "CPAREN",
    "OBRACKET",
    "CBRACKET",
    "COMMA",
    "SYMBOL",
    "COLON",
] + list(reserved.values())

def t_HEX(t):
    r"0x[0-9a-fA-F]+"
    t.value = str(int(t.value, 16))
    return t

def t_INTEGER(t):
    r"\d+"
    t.value = str(int(t.value))
    return t

def t_STR(t):
    r'\"([^\\\n]|(\\.))*?\"|\'([^\\\n]|(\\.))*?\''
    # nip off the quotation marks
    t.value = t.value[1:-1]
    return t

t_EQUALS = r"=="

t_NOTEQUALS = r"!="

t_LT = r"<"

t_GT = r">"

t_LTEQ = r"<="

t_GTEQ = r">="

t_OPAREN = r"[(]"

t_CPAREN = r"[)]"

t_OBRACKET = r"\["

t_CBRACKET = r"\]"

t_COMMA = r","

t_COLON = ":"

def t_SYMBOL(t):
    r"[A-Za-z_][0-9A-Za-z_]*"
    t.type = reserved.get(t.value, "SYMBOL")
    return t

t_ignore = " \t\n"

def t_error(t):
    raise SyntaxError("Unexpected token '%s'" % t.value)

lex.lex()

precedence = (
    ('left', 'OR'),
    ('left', 'AND'),
    ('right', 'NOT'),
    ('nonassoc', 'EQUALS', 'NOTEQUALS', 'GT', 'LT', 'GTEQ', 'LTEQ', 'IN'),
)

def p_expr_or(p):
    'expr : expr OR expr'
    p[0] = ("or", p[1], p[3])

def p_expr_and(p):
    'expr : expr AND expr'
    p[0] = ("and", p[1], p[3])

def p_expr_not(p):
    'expr : NOT expr'
    p[0] = ("not", p[2])

def p_expr_parens(p):
    'expr : OPAREN expr CPAREN'
    p[0] = p[2]

def p_expr_eval(p):
    """expr : SYMBOL EQUALS const
            | SYMBOL NOTEQUALS const
            | SYMBOL GT number
            | SYMBOL LT number
            | SYMBOL GTEQ number
            | SYMBOL LTEQ number
            | SYMBOL IN list
            | SYMBOL COLON STR"""
    p[0] = (p[2], p[1], p[3])

def p_expr_single(p):
    """expr : SYMBOL"""
    p[0] = ("exists", p[1])

def p_func(p):
    """expr : SYMBOL OPAREN arg_intr CPAREN"""
    p[0] = [p[1]]
    p[0].append(p[3])

def p_arg_intr_single(p):
    """arg_intr : const"""
    p[0] = [p[1]]

def p_arg_intr_mult(p):
    """arg_intr : arg_intr COMMA const"""
    p[0] = copy.copy(p[1])
    p[0].append(p[3])

def p_list(p):
    """list : OBRACKET list_intr CBRACKET"""
    p[0] = p[2]

def p_list_intr_single(p):
    """list_intr : const"""
    p[0] = [p[1]]

def p_list_intr_mult(p):
    """list_intr : list_intr COMMA const"""
    p[0] = copy.copy(p[1])
    p[0].append(p[3])

def p_const(p):
    """const : STR
             | number"""
    p[0] = p[1]

def p_number(p):
    """number : INTEGER
              | HEX"""
    p[0] = p[1]

def p_error(p):
    if p:
        raise SyntaxError("Unexpected token '%s'" % p.value)
    else:
        raise SyntaxError("Unexpected end of expression")

if "PARSETAB_DIR" not in os.environ:
    parser = yacc.yacc(debug=0)
else:
    parser = yacc.yacc(debug=0, outputdir=os.environ["PARSETAB_DIR"])

def ast_sym(ast, env):
    if ast in env:
        return str(env[ast])
    return ""

def ast_sym_int(ast, env):
    if ast in env:
        v = env[ast]
        if v.startswith("0x") or v.startswith("0X"):
            return int(v, 16)
        else:
            return int(v, 10)
    return 0

def ast_expr(ast, env, edt):
    if ast[0] == "not":
        return not ast_expr(ast[1], env, edt)
    elif ast[0] == "or":
        return ast_expr(ast[1], env, edt) or ast_expr(ast[2], env, edt)
    elif ast[0] == "and":
        return ast_expr(ast[1], env, edt) and ast_expr(ast[2], env, edt)
    elif ast[0] == "==":
        return ast_sym(ast[1], env) == ast[2]
    elif ast[0] == "!=":
        return ast_sym(ast[1], env) != ast[2]
    elif ast[0] == ">":
        return ast_sym_int(ast[1], env) > int(ast[2])
    elif ast[0] == "<":
        return ast_sym_int(ast[1], env) < int(ast[2])
    elif ast[0] == ">=":
        return ast_sym_int(ast[1], env) >= int(ast[2])
    elif ast[0] == "<=":
        return ast_sym_int(ast[1], env) <= int(ast[2])
    elif ast[0] == "in":
        return ast_sym(ast[1], env) in ast[2]
    elif ast[0] == "exists":
        return bool(ast_sym(ast[1], env))
    elif ast[0] == ":":
        return bool(re.match(ast[2], ast_sym(ast[1], env)))
    elif ast[0] == "dt_compat_enabled":
        compat = ast[1][0]
        for node in edt.nodes:
            if compat in node.compats and node.status == "okay":
                return True
        return False
    elif ast[0] == "dt_alias_exists":
        alias = ast[1][0]
        for node in edt.nodes:
            if alias in node.aliases and node.status == "okay":
                return True
        return False
    elif ast[0] == "dt_enabled_alias_with_parent_compat":
        # Checks if the DT has an enabled alias node whose parent has
        # a given compatible. For matching things like gpio-leds child
        # nodes, which do not have compatibles themselves.
        #
        # The legacy "dt_compat_enabled_with_alias" form is still
        # accepted but is now deprecated and causes a warning. This is
        # meant to give downstream users some time to notice and
        # adjust. Its argument order only made sense under the (bad)
        # assumption that the gpio-leds child node has the same compatible

        alias = ast[1][0]
        compat = ast[1][1]

        return ast_handle_dt_enabled_alias_with_parent_compat(edt, alias,
                                                              compat)
    elif ast[0] == "dt_compat_enabled_with_alias":
        compat = ast[1][0]
        alias = ast[1][1]

        _logger.warning('dt_compat_enabled_with_alias("%s", "%s"): '
                        'this is deprecated, use '
                        'dt_enabled_alias_with_parent_compat("%s", "%s") '
                        'instead',
                        compat, alias, alias, compat)

        return ast_handle_dt_enabled_alias_with_parent_compat(edt, alias,
                                                              compat)
    elif ast[0] == "dt_label_with_parent_compat_enabled":
        compat = ast[1][1]
        label = ast[1][0]
        node = edt.label2node.get(label)
        if node is not None:
            parent = node.parent
        else:
            return False
        return parent is not None and parent.status == 'okay' and parent.matching_compat == compat
    elif ast[0] == "dt_chosen_enabled":
        chosen = ast[1][0]
        node = edt.chosen_node(chosen)
        if node and node.status == "okay":
            return True
        return False

def ast_handle_dt_enabled_alias_with_parent_compat(edt, alias, compat):
    # Helper shared with the now deprecated
    # dt_compat_enabled_with_alias version.

    for node in edt.nodes:
        parent = node.parent
        if parent is None:
            continue
        if (node.status == "okay" and alias in node.aliases and
                    parent.matching_compat == compat):
            return True

    return False

mutex = threading.Lock()

def parse(expr_text, env, edt):
    """Given a text representation of an expression in our language,
    use the provided environment to determine whether the expression
    is true or false"""

    # Like it's C counterpart, state machine is not thread-safe
    mutex.acquire()
    try:
        ast = parser.parse(expr_text)
    finally:
        mutex.release()

    return ast_expr(ast, env, edt)

# Just some test code
if __name__ == "__main__":

    local_env = {
        "A" : "1",
        "C" : "foo",
        "D" : "20",
        "E" : 0x100,
        "F" : "baz"
    }

    for line in open(sys.argv[1]).readlines():
        lex.input(line)
        for tok in iter(lex.token, None):
            print(tok.type, tok.value)

        parser = yacc.yacc()
        print(parser.parse(line))

        print(parse(line, local_env, None))
<commit_msg>twister: Add dt_nodelabel_enabled devicetree filter function<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2016 Intel Corporation.
#
# SPDX-License-Identifier: Apache-2.0

import copy
import logging
import os
import re
import sys
import threading

try:
    import ply.lex as lex
    import ply.yacc as yacc
except ImportError:
    sys.exit("PLY library for Python 3 not installed.\n"
             "Please install the ply package using your workstation's\n"
             "package manager or the 'pip' tool.")

_logger = logging.getLogger('twister')

reserved = {
    'and' : 'AND',
    'or' : 'OR',
    'not' : 'NOT',
    'in' : 'IN',
}

tokens = [
    "HEX",
    "STR",
    "INTEGER",
    "EQUALS",
    "NOTEQUALS",
    "LT",
    "GT",
    "LTEQ",
    "GTEQ",
    "OPAREN",
    "CPAREN",
    "OBRACKET",
    "CBRACKET",
    "COMMA",
    "SYMBOL",
    "COLON",
] + list(reserved.values())

def t_HEX(t):
    r"0x[0-9a-fA-F]+"
    t.value = str(int(t.value, 16))
    return t

def t_INTEGER(t):
    r"\d+"
    t.value = str(int(t.value))
    return t

def t_STR(t):
    r'\"([^\\\n]|(\\.))*?\"|\'([^\\\n]|(\\.))*?\''
    # nip off the quotation marks
    t.value = t.value[1:-1]
    return t

t_EQUALS = r"=="

t_NOTEQUALS = r"!="

t_LT = r"<"

t_GT = r">"

t_LTEQ = r"<="

t_GTEQ = r">="

t_OPAREN = r"[(]"

t_CPAREN = r"[)]"

t_OBRACKET = r"\["

t_CBRACKET = r"\]"

t_COMMA = r","

t_COLON = ":"

def t_SYMBOL(t):
    r"[A-Za-z_][0-9A-Za-z_]*"
    t.type = reserved.get(t.value, "SYMBOL")
    return t

t_ignore = " \t\n"

def t_error(t):
    raise SyntaxError("Unexpected token '%s'" % t.value)

lex.lex()

precedence = (
    ('left', 'OR'),
    ('left', 'AND'),
    ('right', 'NOT'),
    ('nonassoc', 'EQUALS', 'NOTEQUALS', 'GT', 'LT', 'GTEQ', 'LTEQ', 'IN'),
)

def p_expr_or(p):
    'expr : expr OR expr'
    p[0] = ("or", p[1], p[3])

def p_expr_and(p):
    'expr : expr AND expr'
    p[0] = ("and", p[1], p[3])

def p_expr_not(p):
    'expr : NOT expr'
    p[0] = ("not", p[2])

def p_expr_parens(p):
    'expr : OPAREN expr CPAREN'
    p[0] = p[2]

def p_expr_eval(p):
    """expr : SYMBOL EQUALS const
            | SYMBOL NOTEQUALS const
            | SYMBOL GT number
            | SYMBOL LT number
            | SYMBOL GTEQ number
            | SYMBOL LTEQ number
            | SYMBOL IN list
            | SYMBOL COLON STR"""
    p[0] = (p[2], p[1], p[3])

def p_expr_single(p):
    """expr : SYMBOL"""
    p[0] = ("exists", p[1])

def p_func(p):
    """expr : SYMBOL OPAREN arg_intr CPAREN"""
    p[0] = [p[1]]
    p[0].append(p[3])

def p_arg_intr_single(p):
    """arg_intr : const"""
    p[0] = [p[1]]

def p_arg_intr_mult(p):
    """arg_intr : arg_intr COMMA const"""
    p[0] = copy.copy(p[1])
    p[0].append(p[3])

def p_list(p):
    """list : OBRACKET list_intr CBRACKET"""
    p[0] = p[2]

def p_list_intr_single(p):
    """list_intr : const"""
    p[0] = [p[1]]

def p_list_intr_mult(p):
    """list_intr : list_intr COMMA const"""
    p[0] = copy.copy(p[1])
    p[0].append(p[3])

def p_const(p):
    """const : STR
             | number"""
    p[0] = p[1]

def p_number(p):
    """number : INTEGER
              | HEX"""
    p[0] = p[1]

def p_error(p):
    if p:
        raise SyntaxError("Unexpected token '%s'" % p.value)
    else:
        raise SyntaxError("Unexpected end of expression")

if "PARSETAB_DIR" not in os.environ:
    parser = yacc.yacc(debug=0)
else:
    parser = yacc.yacc(debug=0, outputdir=os.environ["PARSETAB_DIR"])

def ast_sym(ast, env):
    if ast in env:
        return str(env[ast])
    return ""

def ast_sym_int(ast, env):
    if ast in env:
        v = env[ast]
        if v.startswith("0x") or v.startswith("0X"):
            return int(v, 16)
        else:
            return int(v, 10)
    return 0

def ast_expr(ast, env, edt):
    if ast[0] == "not":
        return not ast_expr(ast[1], env, edt)
    elif ast[0] == "or":
        return ast_expr(ast[1], env, edt) or ast_expr(ast[2], env, edt)
    elif ast[0] == "and":
        return ast_expr(ast[1], env, edt) and ast_expr(ast[2], env, edt)
    elif ast[0] == "==":
        return ast_sym(ast[1], env) == ast[2]
    elif ast[0] == "!=":
        return ast_sym(ast[1], env) != ast[2]
    elif ast[0] == ">":
        return ast_sym_int(ast[1], env) > int(ast[2])
    elif ast[0] == "<":
        return ast_sym_int(ast[1], env) < int(ast[2])
    elif ast[0] == ">=":
        return ast_sym_int(ast[1], env) >= int(ast[2])
    elif ast[0] == "<=":
        return ast_sym_int(ast[1], env) <= int(ast[2])
    elif ast[0] == "in":
        return ast_sym(ast[1], env) in ast[2]
    elif ast[0] == "exists":
        return bool(ast_sym(ast[1], env))
    elif ast[0] == ":":
        return bool(re.match(ast[2], ast_sym(ast[1], env)))
    elif ast[0] == "dt_compat_enabled":
        compat = ast[1][0]
        for node in edt.nodes:
            if compat in node.compats and node.status == "okay":
                return True
        return False
    elif ast[0] == "dt_alias_exists":
        alias = ast[1][0]
        for node in edt.nodes:
            if alias in node.aliases and node.status == "okay":
                return True
        return False
    elif ast[0] == "dt_enabled_alias_with_parent_compat":
        # Checks if the DT has an enabled alias node whose parent has
        # a given compatible. For matching things like gpio-leds child
        # nodes, which do not have compatibles themselves.
        #
        # The legacy "dt_compat_enabled_with_alias" form is still
        # accepted but is now deprecated and causes a warning. This is
        # meant to give downstream users some time to notice and
        # adjust. Its argument order only made sense under the (bad)
        # assumption that the gpio-leds child node has the same compatible

        alias = ast[1][0]
        compat = ast[1][1]

        return ast_handle_dt_enabled_alias_with_parent_compat(edt, alias,
                                                              compat)
    elif ast[0] == "dt_compat_enabled_with_alias":
        compat = ast[1][0]
        alias = ast[1][1]

        _logger.warning('dt_compat_enabled_with_alias("%s", "%s"): '
                        'this is deprecated, use '
                        'dt_enabled_alias_with_parent_compat("%s", "%s") '
                        'instead',
                        compat, alias, alias, compat)

        return ast_handle_dt_enabled_alias_with_parent_compat(edt, alias,
                                                              compat)
    elif ast[0] == "dt_label_with_parent_compat_enabled":
        compat = ast[1][1]
        label = ast[1][0]
        node = edt.label2node.get(label)
        if node is not None:
            parent = node.parent
        else:
            return False
        return parent is not None and parent.status == 'okay' and parent.matching_compat == compat
    elif ast[0] == "dt_chosen_enabled":
        chosen = ast[1][0]
        node = edt.chosen_node(chosen)
        if node and node.status == "okay":
            return True
        return False
    elif ast[0] == "dt_nodelabel_enabled":
        label = ast[1][0]
        node = edt.label2node.get(label)
        if node and node.status == "okay":
            return True
        return False

def ast_handle_dt_enabled_alias_with_parent_compat(edt, alias, compat):
    # Helper shared with the now deprecated
    # dt_compat_enabled_with_alias version.

    for node in edt.nodes:
        parent = node.parent
        if parent is None:
            continue
        if (node.status == "okay" and alias in node.aliases and
                    parent.matching_compat == compat):
            return True

    return False

mutex = threading.Lock()

def parse(expr_text, env, edt):
    """Given a text representation of an expression in our language,
    use the provided environment to determine whether the expression
    is true or false"""

    # Like it's C counterpart, state machine is not thread-safe
    mutex.acquire()
    try:
        ast = parser.parse(expr_text)
    finally:
        mutex.release()

    return ast_expr(ast, env, edt)

# Just some test code
if __name__ == "__main__":

    local_env = {
        "A" : "1",
        "C" : "foo",
        "D" : "20",
        "E" : 0x100,
        "F" : "baz"
    }

    for line in open(sys.argv[1]).readlines():
        lex.input(line)
        for tok in iter(lex.token, None):
            print(tok.type, tok.value)

        parser = yacc.yacc()
        print(parser.parse(line))

        print(parse(line, local_env, None))
<|endoftext|>


<commit_before>import os, subprocess

import shell

from header import *
from text import Text

def _quit(*args):

    return STOP

def _update(*args):

    print "Updating txtsh..."

    os.chdir(TXTSH_HOME_DIR)
    p = subprocess.Popen(['git', 'pull', 'origin', 'master', '&'])
    code = p.wait()

    if code == 0:
	print "Update complete. Restart txtsh for changes to take effect."

    else:
	print "No update available."

    return GO

def _help(*args):

    print "\r"

    print "--- Main shell commands ---"
    print "!help: display this help screen."
    print "!quit: quit the shell."
    print "!info: display version information."
    print "!update: update txtsh to the latest version."
    print "!load DATA_TYPE, DATA: load DATA into database. DATA_TYPE must be 'string' or 'file'."
    print "!free ID: unload text object stored in ID."
    print "!use ID: go to subshell to manipulate text data stored in ID."
    print "!list: print all loaded objects."

    print "\n--- Subshell commands ---"
    print "!drop: Drop out of subshell created by '!use'."
    print "!words: Display word data for loaded text."
    print "!punct: Display punctuation data for loaded text."
    print "!nums: Display number data for loaded text."

    print "\r"

    return GO

def _info(*args):

    print "TXTSH Text Analysis Shell (ver. 0.0) on %s." % PLATFORM
    print "Type '!help' for a list of commands."

    return GO

def _load(*args):

    """
    data_type must be either 'string' or 'file'.

    If data_type is 'string', data will be
    loaded verbatim into a new Text object.

    If data_type is 'file', it will attempt to
    open the data as a read-only text file and
    load its contents into a new Text object.

    """

    data_type = args[0]
    data = args[1]

    if data_type != 'string' and data_type != 'file':
	print "Error: you must specify 'string' or 'file'"
	return GO

    new = Text()

    try:
	new.load_data(data_type, data)

	return GO

    except:
	print "Loading text data failed."

	return GO

def _free(*args):

    id = int(args[0])

    object = None

    for member in Text.members:
	if member.id == id:

	    object = member

    if not object:
	print "No object found with id '%d'." % id
	return GO

    Text.members.remove(object)

    return GO

def _list(*args):

    if len(Text.members) == 0:
	print "No objects loaded."

    else:
	print "ID\tSAMPLE\n"

	for member in Text.members:
	    print '%2s' % member.id,
	    print '\t',
	    print '"%s"' % member.title

    return GO

def _use(*args):

    id = int(args[0])

    object = None

    for member in Text.members:
	if member.id == id:

	    object = member

    if not object:
	print "No object found with id '%d'." % id
	return GO

    subsh = shell.Subshell(object)
    subsh.run()

    return GO

# ---- MAPPINGS -----

map = {
    '!quit': _quit,
    '!update': _update,
    '!help': _help,
    '!info': _info,
    '!load': _load,
    '!free': _free,
    '!list': _list,
    '!use': _use
       }

<commit_msg>New restart command.<commit_after>import sys, os, subprocess

import shell

from header import *
from text import Text

def _help(*args):

    print "\r"

    print "--- Main shell commands ---"
    print "!help: display this help screen."
    print "!quit: quit the shell."
    print "!info: display version information."
    print "!update: update txtsh to the latest version."
    print "!load DATA_TYPE, DATA: load DATA into database. DATA_TYPE must be 'string' or 'file'."
    print "!free ID: unload text object stored in ID."
    print "!use ID: go to subshell to manipulate text data stored in ID."
    print "!list: print all loaded objects."

    print "\n--- Subshell commands ---"
    print "!drop: Drop out of subshell created by '!use'."
    print "!words: Display word data for loaded text."
    print "!punct: Display punctuation data for loaded text."
    print "!nums: Display number data for loaded text."

    print "\r"

    return GO

def _info(*args):

    print "TXTSH Text Analysis Shell (ver. 0.0) on %s." % PLATFORM
    print "Type '!help' for a list of commands."

    return GO

def _load(*args):

    """
    data_type must be either 'string' or 'file'.

    If data_type is 'string', data will be
    loaded verbatim into a new Text object.

    If data_type is 'file', it will attempt to
    open the data as a read-only text file and
    load its contents into a new Text object.

    """

    data_type = args[0]
    data = args[1]

    if data_type != 'string' and data_type != 'file':
	print "Error: you must specify 'string' or 'file'"
	return GO

    new = Text()

    try:
	new.load_data(data_type, data)

	return GO

    except:
	print "Loading text data failed."

	return GO

def _free(*args):

    id = int(args[0])

    object = None

    for member in Text.members:
	if member.id == id:

	    object = member

    if not object:
	print "No object found with id '%d'." % id
	return GO

    Text.members.remove(object)

    return GO

def _list(*args):

    if len(Text.members) == 0:
	print "No objects loaded."

    else:
	print "ID\tSAMPLE\n"

	for member in Text.members:
	    print '%2s' % member.id,
	    print '\t',
	    print '"%s"' % member.title

    return GO

def _use(*args):

    id = int(args[0])

    object = None

    for member in Text.members:
	if member.id == id:

	    object = member

    if not object:
	print "No object found with id '%d'." % id
	return GO

    subsh = shell.Subshell(object)
    subsh.run()

    return GO

def _quit(*args):

    return STOP

def _restart(*args):

    ans = raw_input("Restart txtsh? All loaded objects will be freed! [y/n]: ")
    if ans.lower().strip() == 'y':

	print "\nRestarting...\n"

	# Free up memory for any loaded objects
	for i in reversed(range(len(Text.members))):
	    _free(Text.members[i].id)

	python = sys.executable

	os.execl(python, python, * sys.argv)

    else:
	return GO

def _update(*args):

    print "Updating txtsh..."

    try:
	os.chdir(TXTSH_HOME_DIR)
	p = subprocess.Popen(['git', 'pull', 'origin', 'master'])
	code = p.wait()

	if code == 0:
	    print "Update complete."
	    print "If changes were made, restart (!restart) txtsh for changes to take effect."

	else:
	    print "Update failed."

    except:
	print "Update command failed to execute."

    return GO

# ---- MAPPINGS -----

map = {
    '!help': _help,
    '!info': _info,
    '!load': _load,
    '!free': _free,
    '!list': _list,
    '!use': _use,
    '!quit': _quit,
    '!restart': _restart,
    '!update': _update
       }

<|endoftext|>


<commit_before>"""
Orthogonal matching pursuit algorithms based on
http://www.cs.technion.ac.il/~ronrubin/Publications/KSVX-OMP-v2.pdf

OMP introduced in S. G. Mallat, Z. Zhang, Matching pursuits with time-frequency
dictionaries, IEEE Transactions on Signal Processing, Vol. 41, No. 12.
(December 1993), pp. 3397-3415.
http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf
"""

# Author: Vlad Niculae
#
# License: BSD Style.

import numpy as np
from scipy import linalg
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..utils.arrayfuncs import solve_triangular

def _cholesky_omp(X, y, n_nonzero_coefs, eps=None):
    """
    Solves a single Orthogonal Matching Pursuit problem using
    the Cholesky decomposition.

    Parameters:
    -----------
    X: array, shape (n_samples, n_features)
        Input dictionary. Columns are assumed to have unit norm.

    y: array, shape: n_samples
        Input targets

    n_nonzero_coefs: int
        Targeted number of non-zero elements

    eps: float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    Returns:
    --------
    gamma: array, shape n_nonzero_coefs
        Non-zero elements of the solution

    idx: array, shape n_nonzero_coefs
        Indices of the positions of the elements in gamma within the solution
        vector

    """

    min_float = np.finfo(X.dtype).eps
    potrs, = get_lapack_funcs(('potrs',), (X,))

    alpha = np.dot(X.T, y)
    residual = y
    n_active = 0
    idx = []

    max_features = X.shape[1] if eps is not None else n_nonzero_coefs
    L = np.empty((max_features, max_features), dtype=X.dtype)
    L[0, 0] = 1.

    while True:
        lam = np.abs(np.dot(X.T, residual)).argmax()
        if lam in idx or alpha[lam] ** 2 < min_float:
            # atom already selected or inner product too small
            break
        if n_active > 0:
            # Updates the Cholesky decomposition of X' X
            L[n_active, :n_active] = np.dot(X[:, idx].T, X[:, lam])
            v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
            solve_triangular(L[:n_active, :n_active], L[n_active, :n_active])
            if 1 - v <= min_float:  # selected atoms are dependent
                break
            L[n_active, n_active] = max(np.sqrt(np.abs(1 - v)), min_float)
        idx.append(lam)
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], alpha[idx], lower=True,
                         overwrite_b=False)

        residual = y - np.dot(X[:, idx], gamma)
        if eps is not None and np.dot(residual.T, residual) <= eps:
            break
        elif n_active == max_features:
            break

    return gamma, idx

def _gram_omp(G, Xy, n_nonzero_coefs, eps_0=None, eps=None):
    """
    Solves a single Orthogonal Matching Pursuit problem using
    the Cholesky decomposition, based on the Gram matrix and more
    precomputations.

    Parameters:
    -----------
    G: array, shape (n_features, n_features)
        Gram matrix of the input data matrix

    Xy: array, shape: n_features
        Input targets

    n_nonzero_coefs: int
        Targeted number of non-zero elements

    eps_0: float
        Squared norm of y, required if eps is not None.

    eps: float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    Returns:
    --------
    gamma: array, shape n_nonzero_coefs
        Non-zero elements of the solution

    idx: array, shape n_nonzero_coefs
        Indices of the positions of the elements in gamma within the solution
        vector

    """

    min_float = np.finfo(G.dtype).eps
    potrs, = get_lapack_funcs(('potrs',), (G,))

    idx = []
    alpha = Xy
    eps_curr = eps_0
    delta = 0
    n_active = 0

    max_features = len(G) if eps is not None else n_nonzero_coefs
    L = np.empty((max_features, max_features), dtype=G.dtype)
    L[0, 0] = 1.

    while True:
        lam = np.abs(alpha).argmax()
        if lam in idx or alpha[lam] ** 2 < min_float:
            # selected same atom twice, or inner product too small
            break
        if n_active > 0:
            L[n_active, :n_active] = G[idx, lam]
            v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
            solve_triangular(L[:n_active, :n_active], L[n_active, :n_active])
            if 1 - v <= min_float:  # selected atoms are dependent
                break
            L[n_active, n_active] = np.sqrt(1 - v)
        idx.append(lam)
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], Xy[idx], lower=True,
                         overwrite_b=False)

        beta = np.dot(G[:, idx], gamma)
        alpha = Xy - beta
        if eps is not None:
            eps_curr += delta
            delta = np.inner(gamma, beta[idx])
            eps_curr -= delta
            if eps_curr <= eps:
                break
        elif n_active == max_features:
            break

    return gamma, idx

def orthogonal_mp(X, y, n_nonzero_coefs=None, eps=None, compute_gram=False):
    """Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems.
    An instance of the problem has the form:

    When parametrized by the number of non-zero coefficients using
    `n_nonzero_coefs`:
    argmin ||y - X\gamma||^2 subject to ||\gamma||_0 <= n_{nonzero coefs}

    When parametrized by error using the parameter `eps`:
    argmin ||\gamma||_0 subject to ||y - X\gamma||^2 <= \epsilon

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Input dictionary. Columns are assumed to have unit norm.

    y: array, shape: n_samples or (n_samples, n_targets)
        Input targets

    n_nonzero_coefs: int
        Desired number of non-zero entries in the solution

    eps: float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    compute_gram, bool:
        Whether to perform precomputations. Improves performance when n_targets
        or n_samples is very large.

    Returns
    -------
    coef: array of shape: n_features or (n_features, n_targets)
        Coefficients of the OMP solution

    See also
    --------
    orthogonal_mp_gram
    lars_path

    """
    X = np.asanyarray(X)
    y = np.asanyarray(y)
    if y.ndim == 1:
        y = y[:, np.newaxis]
    if n_nonzero_coefs == None and eps == None:
        raise ValueError('OMP needs either a target number of atoms \
                         (n_nonzero_coefs) or a target residual error (eps)')
    if eps is not None and eps < 0:
        raise ValueError("Epsilon cannot be negative")
    if eps is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if eps is None and n_nonzero_coefs > X.shape[1]:
        raise ValueError("The number of atoms cannot be more than the number \
                          of features")
    if compute_gram:
        G = np.dot(X.T, X)
        Xy = np.dot(X.T, y)
        if eps is not None:
            norms_squared = np.sum((y ** 2), axis=0)
        else:
            norms_squared = None
        return orthogonal_mp_gram(G, Xy, n_nonzero_coefs, eps, norms_squared)

    coef = np.zeros((X.shape[1], y.shape[1]))
    for k in xrange(y.shape[1]):
        x, idx = _cholesky_omp(X, y[:, k], n_nonzero_coefs, eps)
        coef[idx, k] = x
    return np.squeeze(coef)

def orthogonal_mp_gram(G, Xy, n_nonzero_coefs=None, eps=None,
                       norms_squared=None):
    """Gram Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems using only
    the Gram matrix X.T * X and the product X * y.

    Parameters
    ----------
    G: array of shape (n_features, n_features)
        Gram matrix of the input data: X.T * X

    Xy: array, shape: n_features or (n_features, n_targets)
        Input targets multiplied by X: X * y

    n_nonzero_coefs: int
        Desired number of non-zero entries in the solution

    eps: float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    norms_squared: array, shape: n_targets
        Squared L2 norms of the lines of y. Required if eps is not None.

    Returns
    -------
    coef: array of shape: n_features or (n_features, n_targets)
        Coefficients of the OMP solution

    See also
    --------
    orthogonal_mp
    lars_path

    """
    G = np.asanyarray(G)
    Xy = np.asanyarray(Xy)
    if Xy.ndim == 1:
        Xy = Xy[:, np.newaxis]
        if eps is not None:
            norms_squared = [norms_squared]

    if n_nonzero_coefs == None and eps is None:
        raise ValueError('OMP needs either a target number of atoms \
                         (n_nonzero_coefs) or a target residual error (eps)')
    if eps is not None and norms_squared == None:
        raise ValueError('Gram OMP needs the precomputed norms in order \
                          to evaluate the error sum of squares.')
    if eps is not None and eps < 0:
        raise ValueError("Epsilon cennot be negative")
    if eps is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if eps is None and n_nonzero_coefs > len(G):
        raise ValueError("The number of atoms cannot be more than the number \
                          of features")
    coef = np.zeros((len(G), Xy.shape[1]))
    for k in range(Xy.shape[1]):
        x, idx = _gram_omp(G, Xy[:, k], n_nonzero_coefs,
                           norms_squared[k] if eps is not None else None, eps)
        coef[idx, k] = x
    return np.squeeze(coef)

class OrthogonalMatchingPursuit(LinearModel):
    """Orthogonal Mathching Pursuit model (OMP)

    Parameters
    ----------
    n_nonzero_coefs: int, optional
        number of selected active features

    eps: float, optional
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    fit_intercept: boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize: boolean, optional
        If False, the regressors X are assumed to be already normalized.

    precompute : True | False | array-like, optional
        Whether to use a precomputed Gram matrix to speed up
        calculations. The Gram matrix can also be passed as argument.

    Attributes
    ----------
    `coef_` : array, shape = [n_features]
        parameter vector (w in the fomulation formula)

    `intercept_` : float
        independent term in decision function.

    References
    ----------
    TODO
    """

    def __init__(self, n_nonzero_coefs=None, eps=None, fit_intercept=True,
                 normalize=True, precompute=False):
        self.n_nonzero_coefs = n_nonzero_coefs
        self.eps = eps
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute

    def fit(self, X, y, **params):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data.

        y : array-like, shape = (n_samples,) or (n_samples, n_targets)
            Target values.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        self._set_params(**params)

        X = np.atleast_2d(X)
        y = np.atleast_1d(y)

        X, y, Xmean, ymean = LinearModel._center_data(X, y, self.fit_intercept)
        if self.normalize:
            norms = np.sqrt(np.sum(X ** 2, axis=0))
            nonzeros = np.flatnonzero(norms)
            X[:, nonzeros] /= norms[nonzeros]

        self.coef_ = orthogonal_mp(X, y, self.n_nonzero_coefs, self.eps,
                                   self.precompute).T

        if self.normalize:
            self.coef_ /= norms
        self._set_intercept(Xmean, ymean)
        return self
<commit_msg>n_nonzero_coefs defaults to 10% if eps not passed<commit_after>"""
Orthogonal matching pursuit algorithms based on
http://www.cs.technion.ac.il/~ronrubin/Publications/KSVX-OMP-v2.pdf

OMP introduced in S. G. Mallat, Z. Zhang, Matching pursuits with time-frequency
dictionaries, IEEE Transactions on Signal Processing, Vol. 41, No. 12.
(December 1993), pp. 3397-3415.
http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf
"""

# Author: Vlad Niculae
#
# License: BSD Style.

import numpy as np
from scipy import linalg
from scipy.linalg.lapack import get_lapack_funcs

from .base import LinearModel
from ..utils.arrayfuncs import solve_triangular

def _cholesky_omp(X, y, n_nonzero_coefs, eps=None):
    """
    Solves a single Orthogonal Matching Pursuit problem using
    the Cholesky decomposition.

    Parameters:
    -----------
    X: array, shape (n_samples, n_features)
        Input dictionary. Columns are assumed to have unit norm.

    y: array, shape: n_samples
        Input targets

    n_nonzero_coefs: int
        Targeted number of non-zero elements

    eps: float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    Returns:
    --------
    gamma: array, shape n_nonzero_coefs
        Non-zero elements of the solution

    idx: array, shape n_nonzero_coefs
        Indices of the positions of the elements in gamma within the solution
        vector

    """

    min_float = np.finfo(X.dtype).eps
    potrs, = get_lapack_funcs(('potrs',), (X,))

    alpha = np.dot(X.T, y)
    residual = y
    n_active = 0
    idx = []

    max_features = X.shape[1] if eps is not None else n_nonzero_coefs
    L = np.empty((max_features, max_features), dtype=X.dtype)
    L[0, 0] = 1.

    while True:
        lam = np.abs(np.dot(X.T, residual)).argmax()
        if lam in idx or alpha[lam] ** 2 < min_float:
            # atom already selected or inner product too small
            break
        if n_active > 0:
            # Updates the Cholesky decomposition of X' X
            L[n_active, :n_active] = np.dot(X[:, idx].T, X[:, lam])
            v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
            solve_triangular(L[:n_active, :n_active], L[n_active, :n_active])
            if 1 - v <= min_float:  # selected atoms are dependent
                break
            L[n_active, n_active] = max(np.sqrt(np.abs(1 - v)), min_float)
        idx.append(lam)
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], alpha[idx], lower=True,
                         overwrite_b=False)

        residual = y - np.dot(X[:, idx], gamma)
        if eps is not None and np.dot(residual.T, residual) <= eps:
            break
        elif n_active == max_features:
            break

    return gamma, idx

def _gram_omp(G, Xy, n_nonzero_coefs, eps_0=None, eps=None):
    """
    Solves a single Orthogonal Matching Pursuit problem using
    the Cholesky decomposition, based on the Gram matrix and more
    precomputations.

    Parameters:
    -----------
    G: array, shape (n_features, n_features)
        Gram matrix of the input data matrix

    Xy: array, shape: n_features
        Input targets

    n_nonzero_coefs: int
        Targeted number of non-zero elements

    eps_0: float
        Squared norm of y, required if eps is not None.

    eps: float
        Targeted squared error, if not None overrides n_nonzero_coefs.

    Returns:
    --------
    gamma: array, shape n_nonzero_coefs
        Non-zero elements of the solution

    idx: array, shape n_nonzero_coefs
        Indices of the positions of the elements in gamma within the solution
        vector

    """

    min_float = np.finfo(G.dtype).eps
    potrs, = get_lapack_funcs(('potrs',), (G,))

    idx = []
    alpha = Xy
    eps_curr = eps_0
    delta = 0
    n_active = 0

    max_features = len(G) if eps is not None else n_nonzero_coefs
    L = np.empty((max_features, max_features), dtype=G.dtype)
    L[0, 0] = 1.

    while True:
        lam = np.abs(alpha).argmax()
        if lam in idx or alpha[lam] ** 2 < min_float:
            # selected same atom twice, or inner product too small
            break
        if n_active > 0:
            L[n_active, :n_active] = G[idx, lam]
            v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
            solve_triangular(L[:n_active, :n_active], L[n_active, :n_active])
            if 1 - v <= min_float:  # selected atoms are dependent
                break
            L[n_active, n_active] = np.sqrt(1 - v)
        idx.append(lam)
        n_active += 1
        # solves LL'x = y as a composition of two triangular systems
        gamma, _ = potrs(L[:n_active, :n_active], Xy[idx], lower=True,
                         overwrite_b=False)

        beta = np.dot(G[:, idx], gamma)
        alpha = Xy - beta
        if eps is not None:
            eps_curr += delta
            delta = np.inner(gamma, beta[idx])
            eps_curr -= delta
            if eps_curr <= eps:
                break
        elif n_active == max_features:
            break

    return gamma, idx

def orthogonal_mp(X, y, n_nonzero_coefs=None, eps=None, compute_gram=False):
    """Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems.
    An instance of the problem has the form:

    When parametrized by the number of non-zero coefficients using
    `n_nonzero_coefs`:
    argmin ||y - X\gamma||^2 subject to ||\gamma||_0 <= n_{nonzero coefs}

    When parametrized by error using the parameter `eps`:
    argmin ||\gamma||_0 subject to ||y - X\gamma||^2 <= \epsilon

    Parameters
    ----------
    X: array of shape (n_samples, n_features)
        Input dictionary. Columns are assumed to have unit norm.

    y: array, shape: n_samples or (n_samples, n_targets)
        Input targets

    n_nonzero_coefs: int
        Desired number of non-zero entries in the solution

    eps: float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    compute_gram, bool:
        Whether to perform precomputations. Improves performance when n_targets
        or n_samples is very large.

    Returns
    -------
    coef: array of shape: n_features or (n_features, n_targets)
        Coefficients of the OMP solution

    See also
    --------
    orthogonal_mp_gram
    lars_path

    """
    X = np.asanyarray(X)
    y = np.asanyarray(y)
    if y.ndim == 1:
        y = y[:, np.newaxis]
    if n_nonzero_coefs == None and eps == None:
        n_nonzero_coefs = int(0.1 * X.shape[1])
    if eps is not None and eps < 0:
        raise ValueError("Epsilon cannot be negative")
    if eps is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if eps is None and n_nonzero_coefs > X.shape[1]:
        raise ValueError("The number of atoms cannot be more than the number \
                          of features")
    if compute_gram:
        G = np.dot(X.T, X)
        Xy = np.dot(X.T, y)
        if eps is not None:
            norms_squared = np.sum((y ** 2), axis=0)
        else:
            norms_squared = None
        return orthogonal_mp_gram(G, Xy, n_nonzero_coefs, eps, norms_squared)

    coef = np.zeros((X.shape[1], y.shape[1]))
    for k in xrange(y.shape[1]):
        x, idx = _cholesky_omp(X, y[:, k], n_nonzero_coefs, eps)
        coef[idx, k] = x
    return np.squeeze(coef)

def orthogonal_mp_gram(G, Xy, n_nonzero_coefs=None, eps=None,
                       norms_squared=None):
    """Gram Orthogonal Matching Pursuit (OMP)

    Solves n_targets Orthogonal Matching Pursuit problems using only
    the Gram matrix X.T * X and the product X * y.

    Parameters
    ----------
    G: array of shape (n_features, n_features)
        Gram matrix of the input data: X.T * X

    Xy: array, shape: n_features or (n_features, n_targets)
        Input targets multiplied by X: X * y

    n_nonzero_coefs: int
        Desired number of non-zero entries in the solution

    eps: float
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    norms_squared: array, shape: n_targets
        Squared L2 norms of the lines of y. Required if eps is not None.

    Returns
    -------
    coef: array of shape: n_features or (n_features, n_targets)
        Coefficients of the OMP solution

    See also
    --------
    orthogonal_mp
    lars_path

    """
    G = np.asanyarray(G)
    Xy = np.asanyarray(Xy)
    if Xy.ndim == 1:
        Xy = Xy[:, np.newaxis]
        if eps is not None:
            norms_squared = [norms_squared]

    if n_nonzero_coefs == None and eps is None:
        n_nonzero_coefs = int(0.1 * len(G))
    if eps is not None and norms_squared == None:
        raise ValueError('Gram OMP needs the precomputed norms in order \
                          to evaluate the error sum of squares.')
    if eps is not None and eps < 0:
        raise ValueError("Epsilon cennot be negative")
    if eps is None and n_nonzero_coefs <= 0:
        raise ValueError("The number of atoms must be positive")
    if eps is None and n_nonzero_coefs > len(G):
        raise ValueError("The number of atoms cannot be more than the number \
                          of features")
    coef = np.zeros((len(G), Xy.shape[1]))
    for k in range(Xy.shape[1]):
        x, idx = _gram_omp(G, Xy[:, k], n_nonzero_coefs,
                           norms_squared[k] if eps is not None else None, eps)
        coef[idx, k] = x
    return np.squeeze(coef)

class OrthogonalMatchingPursuit(LinearModel):
    """Orthogonal Mathching Pursuit model (OMP)

    Parameters
    ----------
    n_nonzero_coefs: int, optional
        number of selected active features

    eps: float, optional
        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.

    fit_intercept: boolean, optional
        whether to calculate the intercept for this model. If set
        to false, no intercept will be used in calculations
        (e.g. data is expected to be already centered).

    normalize: boolean, optional
        If False, the regressors X are assumed to be already normalized.

    precompute : True | False | array-like, optional
        Whether to use a precomputed Gram matrix to speed up
        calculations. The Gram matrix can also be passed as argument.

    Attributes
    ----------
    `coef_` : array, shape = [n_features]
        parameter vector (w in the fomulation formula)

    `intercept_` : float
        independent term in decision function.

    References
    ----------
    TODO
    """

    def __init__(self, n_nonzero_coefs=None, eps=None, fit_intercept=True,
                 normalize=True, precompute=False):
        self.n_nonzero_coefs = n_nonzero_coefs
        self.eps = eps
        self.fit_intercept = fit_intercept
        self.normalize = normalize
        self.precompute = precompute

    def fit(self, X, y, **params):
        """Fit the model using X, y as training data.

        Parameters
        ----------
        X : array-like, shape = (n_samples, n_features)
            Training data.

        y : array-like, shape = (n_samples,) or (n_samples, n_targets)
            Target values.

        Returns
        -------
        self : object
            returns an instance of self.
        """
        self._set_params(**params)

        X = np.atleast_2d(X)
        y = np.atleast_1d(y)

        X, y, Xmean, ymean = LinearModel._center_data(X, y, self.fit_intercept)
        if self.normalize:
            norms = np.sqrt(np.sum(X ** 2, axis=0))
            nonzeros = np.flatnonzero(norms)
            X[:, nonzeros] /= norms[nonzeros]

        self.coef_ = orthogonal_mp(X, y, self.n_nonzero_coefs, self.eps,
                                   self.precompute).T

        if self.normalize:
            self.coef_ /= norms
        self._set_intercept(Xmean, ymean)
        return self
<|endoftext|>


<commit_before>#!/usr/bin/python/

"""Wrapper around heapq, to make it an object"""

import heapq

__all__ = ['Heap']

class Heap(object):
    """Wrapper around python's heapq module"""
    def __init__(self, h = None, key = None):
        if h is None:
            self.h = []
        elif key is None:
            self.h = list(h)
        else:
            self.h = [(key(i), i) for i in h]
        heapq.heapify(self.h)
        self.key = key
    def __len__(self):
        return len(self.h)
    def __contains__(self, item):
        if self.key is None:
            return item in self.h
        else:
            for i,j in self.h:
                if item == j:
                    return True
            return False
    def __repr__(self):
        s =  '<' + self.__class__.__name__
        if self.key:
            s += " with key " + self.key.__name__
            s += ": " + ", ".join(repr(j) for i,j in self.h[:10])
        else:
            s += ": " + ", ".join(repr(i) for i in self.h[:10])
        if len(self.h) > 10:
            s += "..."
        s += '>'
        return s
    def pop(self, index = None):
        """Removes the item at index and returns it,
        keeping the heap invariant

        if index is None, pops smallest element"""
        if index is None:
            val =  heapq.heappop(self.h)
        else:
            val = self.h[index]
            self.h[index] = self.h[-1]
            self.h.pop()
            heapq._siftup(self.h, index)
        if self.key:
            return val[1]
        else:
            return val
    def push(self, item):
        if self.key is None:
            heapq.heappush(self.h, item)
        else:
            heapq.heappush(self.h, (self.key(item), item))
    def pushpop(self, item):
        if self.key is None:
            return heapq.heappushpop(self.h, item)
        else:
            return heapq.heappushpop(self.h, (self.key(item), item))[1]
    def heapify(self, key = None):
        """Heapifies the heap. If optional parameter
        key is supplied, uses that as the key"""
        if key is None:
            pass
        elif self.key is None:
            self.h = [(key(i), i) for i in self.h]
        else:
            self.h = [(key(j), j) for i,j in self.h]
        self.key = key
        heapq.heapify(self.h)
    def replace(self, item):
        if self.key is None:
            return heapq.heapreplace(self.h, item)
        else:
            return heapq.heapreplace(self.h, (self.key(item), item))[1]
<commit_msg>more bugfixes<commit_after>#!/usr/bin/python/

"""Wrapper around heapq, to make it an object"""

import heapq

__all__ = ['Heap']

class Heap(object):
    """Wrapper around python's heapq module"""
    def __init__(self, h = None, key = None):
        if h is None:
            self.h = []
        elif key is None:
            self.h = list(h)
        else:
            self.h = [(key(i), i) for i in h]
        heapq.heapify(self.h)
        self.key = key
    def __len__(self):
        return len(self.h)
    def __contains__(self, item):
        if self.key is None:
            return item in self.h
        else:
            for i,j in self.h:
                if item == j:
                    return True
            return False
    def __repr__(self):
        s =  '<' + self.__class__.__name__
        if self.key:
            s += " with key " + self.key.__name__
            s += ": " + ", ".join(repr(j) for i,j in self.h[:10])
        else:
            s += ": " + ", ".join(repr(i) for i in self.h[:10])
        if len(self.h) > 10:
            s += "..."
        s += '>'
        return s
    def pop(self, index = None):
        """Removes the item at index and returns it,
        keeping the heap invariant

        if index is None, pops smallest element"""
        if index is None:
            val =  heapq.heappop(self.h)
        else:
            val = self.h[index]
            self.h[index] = self.h[-1]
            self.h.pop()
            heapq._siftup(self.h, index)
        if self.key:
            return val[1]
        else:
            return val
    def push(self, item):
        if self.key is None:
            heapq.heappush(self.h, item)
        else:
            heapq.heappush(self.h, (self.key(item), item))
    def pushpop(self, item):
        """Push, then pop

        Calls heapq.heappushpop"""
        if self.key is None:
            return heapq.heappushpop(self.h, item)
        else:
            return heapq.heappushpop(self.h, (self.key(item), item))[1]
    def heapify(self, key = None):
        """Heapifies the heap. If optional parameter
        key is supplied, uses that as the key"""
        if key is None:
            pass
        elif self.key is None:
            self.h = [(key(i), i) for i in self.h]
        else:
            self.h = [(key(j), j) for i,j in self.h]
        self.key = key
        heapq.heapify(self.h)
    def replace(self, item):
        """Pop, then push

        Calls heapq.heapreplace"""
        if self.key is None:
            return heapq.heapreplace(self.h, item)
        else:
            return heapq.heapreplace(self.h, (self.key(item), item))[1]
    def consume(self):
        """Outputs the heap as a sorted iterator. Consumes heap while doing so"""
        while self.h:
            yield self.pop()
<|endoftext|>


<commit_before>#!/usr/bin/python
# bdmplot.py
# Main bdmplot generator.

import cgi
import cgitb
cgitb.enable()

# Import matplotlib in such a way that it doesn't leak all over stdout and it
# doesn't wedge when it can't write all over HOME
import os
os.environ [ 'HOME' ] = '/tmp/'
import matplotlib
matplotlib.use('Agg')
import matplotlib.dates as mdates

from timespan import timespan_to_timevals, timeval_to_float
import timechooser

def bdmplot(kwargs, bionet_resources):
    """
    Plots a time series of Bionet Data Manager datapoints on a Matplotlib
    plot, renders that to png, and prints the png to stdout.  If called from
    CGI, the MIME header "Content-Type: image/png" is prepended.

    If this script is placed in a cgi-bin directory of a webserver, users can
    get plots of datapoints contained in your BDM.  Users can pass arguments
    using HTTP GET.  For example, a URL to render all the syshealth-hab's
    uptime datapoints over the last day might look like:

    http://yourwebserver.com/cgi-bin/bdmplot.py?filter=syshealth.*.*:uptime&timespan=last 1d
    """
    # Supply default values
    args = { "label": "plot",
             "filter": "syshealth.*.*:15-min-load-average",
             "timespan": ["last 6h"],
             "regexp": None,
             "format": "png",
             "width": 7,
             "height": 5,
             "dpi": 60,
             "bionet-resources" : {} }

    fname = "/tmp/" + args['filter'][0] + "." + args['timespan'][0]

    # Get args from the caller; these override the defaults but are
    #         overriden by CGI.
    if kwargs != None and len(kwargs) > 0:
        for k,v in kwargs.iteritems():
            args[k] = v

    # Get values from CGI
    # CGI overrides any previous values.
    form = cgi.FieldStorage()
    if len(form) > 0:
        for k in form.keys():
            args[k] = form[k].value

    timespan_vals = timespan_to_timevals(args["timespan"][0])
    timespan_stamps = map(timeval_to_float, timespan_vals)
    timespan_stamps_now = map(lambda x : timeval_to_float(x, evaluateNow = True), timespan_vals)

    # Get the results
    from datapoints_to_dict import datapoints_to_dict
    (updated, results) = datapoints_to_dict(timespan_vals, args["filter"], args["regexp"], bionet_resources)

    # if it hasn't been updated then just use the cached image
    if (updated == False):
        return (fname, args['format'])

    # Split the dictionary into:
    #   - an array of legends based on the keys
    #   - a tuple of args for the lines
    legend = []
    plotargs = []
    if len(results) == 0:
        # If there are no results, matplotlib will barf on an empty set.  Add
        # two datapoints that are off-screen.
        plotargs.append([timespan_stamps_now[0]-100, timespan_stamps_now[0]-99])
        plotargs.append([0, 0])
    else:
        for k,v in results.iteritems():
            legend.append(k)
            vals_T = zip(*v)
            plotargs.append(vals_T[0])
            plotargs.append(vals_T[1])
            # FIXME
            # If the results have a bigger timestamp than our idea about "now", and we wanted to
            # plot until "now", expand the right plot boundary.

    # Plot the results
    import pylab
    f = pylab.figure(figsize=(int(args["width"]), int(args["height"])))
    pylab.step(*plotargs, **{'figure': f})

    # Set the x-axis interval and formatter appropriately
    ax = pylab.gca()
    ax.xaxis.set_major_formatter(timechooser.SmartDateFormatter(timespan_stamps_now))
    pylab.xlim( timespan_stamps_now )

    # Plot 11 major ticks (10 major segments)
    ax.xaxis.set_major_locator(matplotlib.ticker.LinearLocator(numticks=11))
    pylab.xticks(rotation = 33)

    # Render the plot.
    pylab.show()

    pylab.savefig(fname, format=args["format"], dpi=args["dpi"])

    return (fname, args['format'])

if __name__ == "__main__":
    retval = bdmplot()

    # If CGI, add MIME header
    if os.environ.has_key("GATEWAY_INTERFACE"):
        print "Content-Type: image/" + args["format"] + "\n"

    # Print the plot.
    retval['pylab'].savefig(os.sys.stdout, format=retval['args']["format"], dpi=retval['args']["dpi"])
<commit_msg>Generate cached image name only after interpreting kwargs, match twisted convention.<commit_after>#!/usr/bin/python
# bdmplot.py
# Main bdmplot generator.

import cgi
import cgitb
cgitb.enable()

# Import matplotlib in such a way that it doesn't leak all over stdout and it
# doesn't wedge when it can't write all over HOME
import os
os.environ [ 'HOME' ] = '/tmp/'
import matplotlib
matplotlib.use('Agg')
import matplotlib.dates as mdates

from timespan import timespan_to_timevals, timeval_to_float
import timechooser

def bdmplot(kwargs, bionet_resources):
    """
    Plots a time series of Bionet Data Manager datapoints on a Matplotlib
    plot, renders that to png, and prints the png to stdout.  If called from
    CGI, the MIME header "Content-Type: image/png" is prepended.

    If this script is placed in a cgi-bin directory of a webserver, users can
    get plots of datapoints contained in your BDM.  Users can pass arguments
    using HTTP GET.  For example, a URL to render all the syshealth-hab's
    uptime datapoints over the last day might look like:

    http://yourwebserver.com/cgi-bin/bdmplot.py?filter=syshealth.*.*:uptime&timespan=last 1d
    """
    # Supply default values
    args = { "label": "plot",
             "filter": ["syshealth.*.*:15-min-load-average"],
             "timespan": ["last 6h"],
             "regexp": None,
             "format": "png",
             "width": 7,
             "height": 5,
             "dpi": 60,
             "bionet-resources" : {} }

    # Get args from the caller; these override the defaults but are
    #         overriden by CGI.
    if kwargs != None and len(kwargs) > 0:
        for k,v in kwargs.iteritems():
            args[k] = v

    fname = "/tmp/" + args['filter'] + "." + args['timespan'][0]

    # Get values from CGI
    # CGI overrides any previous values.
    form = cgi.FieldStorage()
    if len(form) > 0:
        for k in form.keys():
            args[k] = form[k].value

    timespan_vals = timespan_to_timevals(args["timespan"][0])
    timespan_stamps = map(timeval_to_float, timespan_vals)
    timespan_stamps_now = map(lambda x : timeval_to_float(x, evaluateNow = True), timespan_vals)

    # Get the results
    from datapoints_to_dict import datapoints_to_dict
    (updated, results) = datapoints_to_dict(timespan_vals, args["filter"], args["regexp"], bionet_resources)

    # if it hasn't been updated then just use the cached image
    if (updated == False):
        return (fname, args['format'])

    # Split the dictionary into:
    #   - an array of legends based on the keys
    #   - a tuple of args for the lines
    legend = []
    plotargs = []
    if len(results) == 0:
        # If there are no results, matplotlib will barf on an empty set.  Add
        # two datapoints that are off-screen.
        plotargs.append([timespan_stamps_now[0]-100, timespan_stamps_now[0]-99])
        plotargs.append([0, 0])
    else:
        for k,v in results.iteritems():
            legend.append(k)
            vals_T = zip(*v)
            plotargs.append(vals_T[0])
            plotargs.append(vals_T[1])
            # FIXME
            # If the results have a bigger timestamp than our idea about "now", and we wanted to
            # plot until "now", expand the right plot boundary.

    # Plot the results
    import pylab
    f = pylab.figure(figsize=(int(args["width"]), int(args["height"])))
    pylab.step(*plotargs, **{'figure': f})

    # Set the x-axis interval and formatter appropriately
    ax = pylab.gca()
    ax.xaxis.set_major_formatter(timechooser.SmartDateFormatter(timespan_stamps_now))
    pylab.xlim( timespan_stamps_now )

    # Plot 11 major ticks (10 major segments)
    ax.xaxis.set_major_locator(matplotlib.ticker.LinearLocator(numticks=11))
    pylab.xticks(rotation = 33)

    # Render the plot.
    pylab.show()

    pylab.savefig(fname, format=args["format"], dpi=args["dpi"])

    return (fname, args['format'])

if __name__ == "__main__":
    retval = bdmplot()

    # If CGI, add MIME header
    if os.environ.has_key("GATEWAY_INTERFACE"):
        print "Content-Type: image/" + args["format"] + "\n"

    # Print the plot.
    retval['pylab'].savefig(os.sys.stdout, format=retval['args']["format"], dpi=retval['args']["dpi"])
<|endoftext|>


<commit_before>import dominate
import itertools
import copy
import bunch
import yaml
from datetime import datetime
from github import Github
import argparse
from dominate.tags import table, tbody, h3
import report
import smtplib
from email.mime.text import MIMEText

def make_email_body(issues):
    title = 'BETA:\nPlease update or close the following issues so Bianca and Spencer know what is going on.'
    d = dominate.document(title=title)
    with d.body:
        h3(title)
        with table(border='1', width='1024', cellpadding='10').add(tbody()):
            report.make_table_header()
            decorated_issues = report.build_decorated_issues(issues)
            decorated_issues.sort(key=lambda issue: issue.priority)
            for decorated_issue in decorated_issues:
                report.make_table_row(decorated_issue)
    return str(d)

def get_notification_from_repos(criteria, repos):
    notification_lists = [get_notification_from_issues(criteria, repo.get_issues()) for repo in repos]
    notifications = list(itertools.chain.from_iterable(notification_lists))
    return notifications

def time_to_notify(timeframe, last_updated):
    """I'm making this a seperate function, as I think we will eventually find some
    odd edge cases that we will want to build into this logic (last updated 23.5 hours ago, etc).
    target_days (int or float): number of days at which we want to notify
    last_updated (datetime): datetime object from github of last time issues was updated """
    day_delta = (datetime.now() - last_updated).days
    if 'catchup' in timeframe and timeframe.catchup:
        return day_delta >= timeframe.days
    else:
        return day_delta == timeframe.days

def generate_recipients(timeframe, issue):
    recipients = copy.deepcopy(timeframe.recipients)
    if 'AUTHORS' in recipients:
        def debug_assignees(assignee):
            if assignee.email is None:
                print('''Github user {} can't be contacted as they don't
                      have a public email set. Found in issue {}.'''.format(assignee, issue))
        list(map(debug_assignees, issue.assignees))
        assignee_emails = [assignee.email for assignee in issue.assignees if assignee.email]
        if not assignee_emails:
            issue.add_to_labels('triage')
        recipients.remove('AUTHORS')
        recipients += assignee_emails
    return list(set(recipients))  # removing duplicates

def get_notification_from_issues(criteria, issues):
    notifications = []
    notification_labels = set(criteria.keys())
    for issue in issues:
        issue_labels = set(str(label.name) for label in issue.labels)
        target_labels = issue_labels & notification_labels
        for target_label in target_labels:
            # reversed sorted order so if there is a different notification level at 60 and one at 30, only the 60 is sent
            for timeframe in sorted(criteria[target_label], key=lambda x: x.days, reverse=True):
                if time_to_notify(timeframe, issue.updated_at):
                    recipients = generate_recipients(timeframe, issue)
                    if recipients:
                        notifications.append({'issue': issue, 'recipients': recipients})
                    break
    return notifications

def process_notification_data(notification_data):
    if 'severity' not in notification_data:
        notification_data[notification_data] = {'default': ['AUTHOR']}
    if 'default_severity' not in notification_data:
        only_one_to_choose_from = 'severity' in notification_data and len(notification_data['severity']) == 1
        if only_one_to_choose_from:
            notification_data['default_severity'] = notification_data['severity'].keys()[0]
    if 'criteria' in notification_data:
        for label, timeframes in notification_data['criteria'].items():
            for timeframe in timeframes:
                if 'severity' in timeframe:
                    severity = timeframe['severity']
                else:
                    severity = notification_data['default_severity']
                timeframe['recipients'] = notification_data['severity'][severity]
                if 'catchup' in notification_data and notification_data['catchup'] and 'catchup' not in timeframe:
                    timeframe['catchup'] = True
                if 'days_multiplier' in notification_data:
                    timeframe['days'] = int(notification_data['days_multiplier'] * timeframe['days'])
            notification_data['criteria'][label] = list(map(bunch.Bunch, timeframes))
    else:
        raise(Exception('This system cannot function without notification "criteria" being specified'))
    return notification_data['criteria']

def get_notification_criteria(config_file_path):
    # todo: should use a schema file to make sure format is correct
    with open(config_file_path, 'r') as yaml_file:
        notification_data = yaml.load(yaml_file)
    criteria = process_notification_data(notification_data)
    return criteria

def sort_issue_notifications_into_emails(issue_notifications):
    notification_groups = {}
    for notification in issue_notifications:
        recipients = ''.join(sorted(notification['recipients']))
        notification_groups[recipients] = notification_groups.get(recipients, []) + [notification]
    emails = []
    for note_group in notification_groups.values():
        issues = [notification['issue'] for notification in note_group]
        emails.append(bunch.Bunch({'to': note_group[0]['recipients'], 'body': make_email_body(issues),
                                   'issues': issues}))
    return emails

def print_email_debug(emails):
    print('Email Summary\n(Add --send flag to send out emails)')
    print('{} emails containing {} issues\n'.format(len(emails), sum(map(lambda x: len(x.issues), emails))))
    for email in emails:
        issue_titles = '\n    '.join('"{}" {}'.format(issue.title, issue.html_url) for issue in email.issues)
        print('recipients {} are getting {} emails:\n    {}'.format(email.to, len(email.issues), issue_titles))

def send_email(subject, body, to=[], cc=[], bcc=[]):
    """
    This routine takes basic components, compiles and sends an email
    :param body: string containing HTML text to be sent
    :param to: LIST of subjects to mail to
    :param cc: LIST of subjects to cc
    :param bcc: LIST of subjects to bcc
    :return: None
    """
    msg = MIMEText(body, 'html')
    msg['Subject'] = subject
    email_from = 'github_msg_bot@bot_machine.com'  # todo: figure this out
    msg['From'] = email_from
    if cc:
        msg['Cc'] = ', '.join(cc)

    server = smtplib.SMTP('localhost')
    server.sendmail(msg['From'], to + cc + bcc, msg.as_string())

def get_arg_parser():
    parser = argparse.ArgumentParser(description='Sends email reminders for Github Issues that need updates.')
    parser.add_argument('-s', '--send', dest='send', action='store_true', help="print email details but don't send")
    parser.add_argument('github_token', type=str, help='Github authentication token')
    parser.add_argument('github_repo', type=str, help='Target Github Repository')
    parser.add_argument('github_org', type=str, help='Target Github Organization')
    parser.add_argument('config_file_path', type=str, nargs='?', help='Full-path to config file', default='config.yaml')
    args = parser.parse_args()
    return args

def main(args=None):
    if args is None:
        args = get_arg_parser()
    token = args.github_token
    repository = args.github_repo
    organization = args.github_org
    config_file_path = args.config_file_path
    g = Github(token)
    repos = g.get_organization(organization).get_repos(repository)

    criteria = get_notification_criteria(config_file_path)
    issue_notifications = get_notification_from_repos(criteria, repos)
    emails = sort_issue_notifications_into_emails(issue_notifications)
    if args.send:
        for email in emails:
            send_email('Please Update the Following Github Issues', email.body, email.to)
    else:
        print_email_debug(emails)

if __name__ == '__main__':
    main()
<commit_msg>expanding email prompt to include downgrading issues<commit_after>import dominate
import itertools
import copy
import bunch
import yaml
from datetime import datetime
from github import Github
import argparse
from dominate.tags import table, tbody, h3
import report
import smtplib
from email.mime.text import MIMEText

def make_email_body(issues):
    title = 'BETA:\nPlease update, close or change the priority of the following issues so your team knows what is going on.'
    d = dominate.document(title=title)
    with d.body:
        h3(title)
        with table(border='1', width='1024', cellpadding='10').add(tbody()):
            report.make_table_header()
            decorated_issues = report.build_decorated_issues(issues)
            decorated_issues.sort(key=lambda issue: issue.priority)
            for decorated_issue in decorated_issues:
                report.make_table_row(decorated_issue)
    return str(d)

def get_notification_from_repos(criteria, repos):
    notification_lists = [get_notification_from_issues(criteria, repo.get_issues()) for repo in repos]
    notifications = list(itertools.chain.from_iterable(notification_lists))
    return notifications

def time_to_notify(timeframe, last_updated):
    """I'm making this a seperate function, as I think we will eventually find some
    odd edge cases that we will want to build into this logic (last updated 23.5 hours ago, etc).
    target_days (int or float): number of days at which we want to notify
    last_updated (datetime): datetime object from github of last time issues was updated """
    day_delta = (datetime.now() - last_updated).days
    if 'catchup' in timeframe and timeframe.catchup:
        return day_delta >= timeframe.days
    else:
        return day_delta == timeframe.days

def generate_recipients(timeframe, issue):
    recipients = copy.deepcopy(timeframe.recipients)
    if 'AUTHORS' in recipients:
        def debug_assignees(assignee):
            if assignee.email is None:
                print('''Github user {} can't be contacted as they don't
                      have a public email set. Found in issue {}.'''.format(assignee, issue))
        list(map(debug_assignees, issue.assignees))
        assignee_emails = [assignee.email for assignee in issue.assignees if assignee.email]
        if not assignee_emails:
            issue.add_to_labels('triage')
        recipients.remove('AUTHORS')
        recipients += assignee_emails
    return list(set(recipients))  # removing duplicates

def get_notification_from_issues(criteria, issues):
    notifications = []
    notification_labels = set(criteria.keys())
    for issue in issues:
        issue_labels = set(str(label.name) for label in issue.labels)
        target_labels = issue_labels & notification_labels
        for target_label in target_labels:
            # reversed sorted order so if there is a different notification level at 60 and one at 30, only the 60 is sent
            for timeframe in sorted(criteria[target_label], key=lambda x: x.days, reverse=True):
                if time_to_notify(timeframe, issue.updated_at):
                    recipients = generate_recipients(timeframe, issue)
                    if recipients:
                        notifications.append({'issue': issue, 'recipients': recipients})
                    break
    return notifications

def process_notification_data(notification_data):
    if 'severity' not in notification_data:
        notification_data[notification_data] = {'default': ['AUTHOR']}
    if 'default_severity' not in notification_data:
        only_one_to_choose_from = 'severity' in notification_data and len(notification_data['severity']) == 1
        if only_one_to_choose_from:
            notification_data['default_severity'] = notification_data['severity'].keys()[0]
    if 'criteria' in notification_data:
        for label, timeframes in notification_data['criteria'].items():
            for timeframe in timeframes:
                if 'severity' in timeframe:
                    severity = timeframe['severity']
                else:
                    severity = notification_data['default_severity']
                timeframe['recipients'] = notification_data['severity'][severity]
                if 'catchup' in notification_data and notification_data['catchup'] and 'catchup' not in timeframe:
                    timeframe['catchup'] = True
                if 'days_multiplier' in notification_data:
                    timeframe['days'] = int(notification_data['days_multiplier'] * timeframe['days'])
            notification_data['criteria'][label] = list(map(bunch.Bunch, timeframes))
    else:
        raise(Exception('This system cannot function without notification "criteria" being specified'))
    return notification_data['criteria']

def get_notification_criteria(config_file_path):
    # todo: should use a schema file to make sure format is correct
    with open(config_file_path, 'r') as yaml_file:
        notification_data = yaml.load(yaml_file)
    criteria = process_notification_data(notification_data)
    return criteria

def sort_issue_notifications_into_emails(issue_notifications):
    notification_groups = {}
    for notification in issue_notifications:
        recipients = ''.join(sorted(notification['recipients']))
        notification_groups[recipients] = notification_groups.get(recipients, []) + [notification]
    emails = []
    for note_group in notification_groups.values():
        issues = [notification['issue'] for notification in note_group]
        emails.append(bunch.Bunch({'to': note_group[0]['recipients'], 'body': make_email_body(issues),
                                   'issues': issues}))
    return emails

def print_email_debug(emails):
    print('Email Summary\n(Add --send flag to send out emails)')
    print('{} emails containing {} issues\n'.format(len(emails), sum(map(lambda x: len(x.issues), emails))))
    for email in emails:
        issue_titles = '\n    '.join('"{}" {}'.format(issue.title, issue.html_url) for issue in email.issues)
        print('recipients {} are getting {} emails:\n    {}'.format(email.to, len(email.issues), issue_titles))

def send_email(subject, body, to=[], cc=[], bcc=[]):
    """
    This routine takes basic components, compiles and sends an email
    :param body: string containing HTML text to be sent
    :param to: LIST of subjects to mail to
    :param cc: LIST of subjects to cc
    :param bcc: LIST of subjects to bcc
    :return: None
    """
    msg = MIMEText(body, 'html')
    msg['Subject'] = subject
    email_from = 'github_msg_bot@bot_machine.com'  # todo: figure this out
    msg['From'] = email_from
    if cc:
        msg['Cc'] = ', '.join(cc)

    server = smtplib.SMTP('localhost')
    server.sendmail(msg['From'], to + cc + bcc, msg.as_string())

def get_arg_parser():
    parser = argparse.ArgumentParser(description='Sends email reminders for Github Issues that need updates.')
    parser.add_argument('-s', '--send', dest='send', action='store_true', help="print email details but don't send")
    parser.add_argument('github_token', type=str, help='Github authentication token')
    parser.add_argument('github_repo', type=str, help='Target Github Repository')
    parser.add_argument('github_org', type=str, help='Target Github Organization')
    parser.add_argument('config_file_path', type=str, nargs='?', help='Full-path to config file', default='config.yaml')
    args = parser.parse_args()
    return args

def main(args=None):
    if args is None:
        args = get_arg_parser()
    token = args.github_token
    repository = args.github_repo
    organization = args.github_org
    config_file_path = args.config_file_path
    g = Github(token)
    repos = g.get_organization(organization).get_repos(repository)

    criteria = get_notification_criteria(config_file_path)
    issue_notifications = get_notification_from_repos(criteria, repos)
    emails = sort_issue_notifications_into_emails(issue_notifications)
    if args.send:
        for email in emails:
            send_email('Please Update the Following Github Issues', email.body, email.to)
    else:
        print_email_debug(emails)

if __name__ == '__main__':
    main()
<|endoftext|>