<commit_before>import unittest
import os
from snoohelper.utils.teams import SlackTeamsController
from snoohelper.utils.slack import SlackResponse, SlackRequest
from snoohelper.utils.credentials import get_token
from snoohelper.reddit.bot_modules.flair_enforcer import UnflairedSubmission
from snoohelper.webapp.requests_handler import RequestsHandler
from snoohelper.webapp.webapp import create_app

def create_dummy_command_request(command):
    dummy_request = {
        'user_name': 'Santi871',
        'team_domain': 'snoohelpertesting',
        'team_id': get_token("team_id", "SnooHelper Testing", "teams_test.ini"),
        'command': command,
        'text': 'santi871',
        'channel_name': '#general',
        'response_url': get_token("webhook_url", "SnooHelper Testing", "teams_test.ini"),
        'token': 'abc123'
    }
    return SlackRequest(form=dummy_request)

class SnooHelperTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.team_name = "SnooHelper Testing"
        travis_ci = os.environ.get('team_name', False)
        if travis_ci:
            cls.teams_controller = SlackTeamsController("teams_test.ini", "snoohelper_test.db", vars_from_env=True)
        else:
            cls.teams_controller = SlackTeamsController("teams_test.ini", "snoohelper_test.db")
        cls.bot = cls.teams_controller.teams[cls.team_name].bot
        cls.submission = cls.bot.r.submission("5gk734")
        cls.requests_handler = RequestsHandler(cls.teams_controller)
        app = create_app(cls.teams_controller, cls.requests_handler)
        cls.app = app.test_client()
        cls.app.testing = True

    def test_botban(self):
        response = self.bot.botban("santi871", "santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_unbotban(self):
        response = self.bot.unbotban("santi871", "santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_track_user(self):
        response = self.bot.track_user("santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_check_user_offenses(self):
        self.bot.user_warnings.check_user_offenses('santi871')

    def test_send_warning(self):
        self.bot.user_warnings.send_warning(self.submission)

    def test_check_user_posts(self):
        self.bot.user_warnings.check_user_posts(self.submission)

    def test_untrack_user(self):
        response = self.bot.untrack_user("santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_add_unflaired_submission(self):
        submission = self.bot.flair_enforcer.add_submission(self.submission)
        self.assertTrue(isinstance(submission, UnflairedSubmission))

    def test_quick_summary(self):
        self.bot.summary_generator.generate_quick_summary('santi871')

    def test_expanded_summary(self):
        self.bot.summary_generator.generate_expanded_summary('santi871', 100)

    def test_command_requests(self):
        dummy_request = create_dummy_command_request('/summary')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response, SlackResponse))

        dummy_request = create_dummy_command_request('/botban')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response,SlackResponse))

        dummy_request = create_dummy_command_request('/modmail')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response, SlackResponse))

    def test_slack_auth_endpoint(self):
        result = self.app.get("/slack/oauthcallback")
        self.assertEqual(result.status_code, 302)

    def test_reddit_auth_endpoint(self):
        result = self.app.get("/reddit/oauthcallback")
        self.assertEqual(result.status_code, 302)

if __name__ == '__main__':
    unittest.main()
<commit_msg>fixed team name<commit_after>import unittest
import os
from snoohelper.utils.teams import SlackTeamsController
from snoohelper.utils.slack import SlackResponse, SlackRequest
from snoohelper.utils.credentials import get_token
from snoohelper.reddit.bot_modules.flair_enforcer import UnflairedSubmission
from snoohelper.webapp.requests_handler import RequestsHandler
from snoohelper.webapp.webapp import create_app

def create_dummy_command_request(command):
    dummy_request = {
        'user_name': 'Santi871',
        'team_domain': 'snoohelpertesting',
        'team_id': get_token("team_id", "SnooHelper Testing", "teams_test.ini"),
        'command': command,
        'text': 'santi871',
        'channel_name': '#general',
        'response_url': get_token("webhook_url", "SnooHelper Testing", "teams_test.ini"),
        'token': 'abc123'
    }
    return SlackRequest(form=dummy_request)

class SnooHelperTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.team_name = "SnooHelper"
        travis_ci = os.environ.get('team_name', False)
        if travis_ci:
            cls.teams_controller = SlackTeamsController("teams_test.ini", "snoohelper_test.db", vars_from_env=True)
        else:
            cls.teams_controller = SlackTeamsController("teams_test.ini", "snoohelper_test.db")
        cls.bot = cls.teams_controller.teams[cls.team_name].bot
        cls.submission = cls.bot.r.submission("5gk734")
        cls.requests_handler = RequestsHandler(cls.teams_controller)
        app = create_app(cls.teams_controller, cls.requests_handler)
        cls.app = app.test_client()
        cls.app.testing = True

    def test_botban(self):
        response = self.bot.botban("santi871", "santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_unbotban(self):
        response = self.bot.unbotban("santi871", "santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_track_user(self):
        response = self.bot.track_user("santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_check_user_offenses(self):
        self.bot.user_warnings.check_user_offenses('santi871')

    def test_send_warning(self):
        self.bot.user_warnings.send_warning(self.submission)

    def test_check_user_posts(self):
        self.bot.user_warnings.check_user_posts(self.submission)

    def test_untrack_user(self):
        response = self.bot.untrack_user("santi871")
        self.assertTrue(isinstance(response, SlackResponse))

    def test_add_unflaired_submission(self):
        submission = self.bot.flair_enforcer.add_submission(self.submission)
        self.assertTrue(isinstance(submission, UnflairedSubmission))

    def test_quick_summary(self):
        self.bot.summary_generator.generate_quick_summary('santi871')

    def test_expanded_summary(self):
        self.bot.summary_generator.generate_expanded_summary('santi871', 100)

    def test_command_requests(self):
        dummy_request = create_dummy_command_request('/summary')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response, SlackResponse))

        dummy_request = create_dummy_command_request('/botban')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response,SlackResponse))

        dummy_request = create_dummy_command_request('/modmail')
        response = self.requests_handler.handle_command(dummy_request)
        self.assertTrue(isinstance(response, SlackResponse))

    def test_slack_auth_endpoint(self):
        result = self.app.get("/slack/oauthcallback")
        self.assertEqual(result.status_code, 302)

    def test_reddit_auth_endpoint(self):
        result = self.app.get("/reddit/oauthcallback")
        self.assertEqual(result.status_code, 302)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import ast
import opcode
import threading
import traceback
from collections import defaultdict
from datetime import datetime
from textwrap import dedent

import executing
import six
from pygments import highlight
from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.lexers.python import Python3Lexer, PythonLexer
from six import PY3

from snoop.utils import ensure_tuple, short_filename, with_needed_parentheses, my_cheap_repr, \
    NO_ASTTOKENS, optional_numeric_label, try_statement, FormattedValue, ArgDefaultDict, lru_cache

class StatementsDict(dict):
    def __init__(self, source):
        super(StatementsDict, self).__init__()
        self.source = source

    def __missing__(self, key):
        statements = self.source.statements_at_line(key)
        if len(statements) == 1:
            result = list(statements)[0]
        else:
            result = None
        self[key] = result
        return result

class Source(executing.Source):
    def __init__(self, *args, **kwargs):
        super(Source, self).__init__(*args, **kwargs)
        if self.tree:
            self.lines = self.text.splitlines()
            self.highlighted = ArgDefaultDict(
                lambda style: raw_highlight(self.text, style).splitlines()
            )
        else:
            self.lines = defaultdict(lambda: u'SOURCE IS UNAVAILABLE')
            self.highlighted = defaultdict(lambda: self.lines)
        self.statements = StatementsDict(self)
        self.nodes = []
        if self.tree:
            self.tree._depth = 0
            for node in ast.walk(self.tree):
                node._tree_index = len(self.nodes)
                self.nodes.append(node)
                for child in ast.iter_child_nodes(node):
                    child._depth = node._depth + 1

    def get_text_with_indentation(self, node):
        result = self.asttokens().get_text(node)

        if not result:
            if isinstance(node, FormattedValue):
                fvals = [
                    n for n in node.parent.values
                    if isinstance(n, FormattedValue)
                ]
                return '<f-string value{}>'.format(
                    optional_numeric_label(
                        fvals.index(node),
                        fvals,
                    )
                )
            else:
                return "<unknown>"

        if '\n' in result:
            result = ' ' * node.first_token.start[1] + result
            result = dedent(result)
        else:
            result = result.strip()
        return result

if PY3:
    lexer = Python3Lexer()
else:
    lexer = PythonLexer()

class ForceWhiteTerminal256Formatter(Terminal256Formatter):
    def _closest_color(self, r, g, b):
        result = super(ForceWhiteTerminal256Formatter, self)._closest_color(r, g, b)
        if result == 15:
            result += 6 ** 3
        return result

formatters = ArgDefaultDict(lambda style: ForceWhiteTerminal256Formatter(style=style))

def raw_highlight(code, style):
    return highlight(code, lexer, formatters[style])

cached_highlight = lru_cache(maxsize=1024)(raw_highlight)

class Event(object):
    def __init__(self, frame_info, event, arg, depth, line_no=None):
        self.frame_info = frame_info
        self.frame = frame = frame_info.frame
        self.source = frame_info.source
        self.last_line_no = frame_info.last_line_no
        self.comprehension_type = frame_info.comprehension_type

        self.event = event
        self.arg = arg
        self.depth = depth

        self.variables = []
        if line_no is None:
            line_no = frame.f_lineno
        self.line_no = line_no
        self.code = frame.f_code

        if self.event == 'call' and self.source_line.lstrip().startswith('@'):
            # If a function decorator is found, skip lines until an actual
            # function definition is found.
            while True:
                self.line_no += 1
                try:
                    if self.source_line.lstrip().startswith('def'):
                        break
                except IndexError:
                    self.line_no = self.frame.lineno
                    break

    @property
    def source_line(self):
        return self.source.lines[self.line_no - 1]

    def code_qualname(self):
        return self.source.code_qualname(self.code)

    @property
    def opname(self):
        code_byte = self.code.co_code[self.frame.f_lasti]
        if not isinstance(code_byte, int):
            code_byte = ord(code_byte)
        return opcode.opname[code_byte]

class DefaultFormatter(object):
    datetime_format = None

    def __init__(self, prefix, columns, color):
        prefix = six.text_type(prefix)
        if prefix and prefix == prefix.rstrip():
            prefix += ' '
        self.prefix = prefix
        self.columns = [
            column if callable(column) else
            getattr(self, '{}_column'.format(column))
            for column in ensure_tuple(columns, split=True)
        ]
        self.column_widths = dict.fromkeys(self.columns, 0)
        if color is True:
            color = "monokai"
        if color:
            self.c = Colors

            def highlighted(code):
                return cached_highlight(code, color)

            def highlighted_source_line(event):
                return event.source.highlighted[color][event.line_no - 1]
        else:
            self.c = NoColors()

            def highlighted(code):
                return code

            def highlighted_source_line(event):
                return event.source_line

        self.highlighted = highlighted
        self.highlighted_source_line = highlighted_source_line

    def thread_column(self, _event):
        return threading.current_thread().name

    def thread_ident_column(self, _event):
        return threading.current_thread().ident

    def time_column(self, _event):
        datetime_format = self.datetime_format or '%H:%M:%S.%f'
        result = datetime.now().strftime(datetime_format)
        if self.datetime_format is None:
            result = result[:-4]
        return result

    def file_column(self, event):
        return short_filename(event.code)

    def full_file_column(self, event):
        return _get_filename(event)

    def function_column(self, event):
        return event.code.co_name

    def function_qualname_column(self, event):
        return event.code_qualname()

    def full_prefix(self, event):
        return u'{c.grey}{self.prefix}{indent}{columns} {c.reset}'.format(
            c=self.c,
            self=self,
            indent=event.depth * u'    ',
            columns=self.columns_string(event),
        )

    def format(self, event):
        # type: (Event) -> str
        lines = []

        if event.event in ('call', 'enter'):
            lines += self.format_start(event)

        statements = event.source.statements
        this_statement = statements[event.line_no]
        last_statement = statements[event.last_line_no]
        statement_start_lines = self.get_statement_start_lines(event, this_statement, last_statement)

        lines += self.format_variables(event, last_statement)

        if event.event == 'return':
            lines += self.format_return(event)
        elif event.event == 'exception':
            lines += self.format_exception(event)
        elif event.event == 'enter':
            pass
        elif event.event == 'exit':
            lines += [u'{c.green}<<< Exit with block in {func}{c.reset}'.format(
                c=self.c,
                func=event.code_qualname(),
            )]
        else:
            if not (event.comprehension_type and event.event == 'line'):
                lines += statement_start_lines + [self.format_event(event)]

        return self.format_lines(event, lines)

    def format_exception(self, event):
        lines = []
        exception_string = ''.join(traceback.format_exception_only(*event.arg[:2]))
        lines += [
            u'{c.red}!!! {line}{c.reset}'.format(
                c=self.c,
                line=line,
            )
            for line in exception_string.splitlines()
        ]
        lines += self.format_executing_node_exception(event)
        return lines

    def format_return(self, event):
        # If a call ends due to an exception, we still get a 'return' event
        # with arg = None. This seems to be the only way to tell the difference
        # https://stackoverflow.com/a/12800909/2482744
        opname = event.opname
        arg = event.arg
        if arg is None:
            if opname == 'END_FINALLY':
                if event.frame_info.had_exception:
                    return [u'{c.red}??? Call either returned None or ended by exception{c.reset}'
                                .format(c=self.c)]
            elif opname not in ('RETURN_VALUE', 'YIELD_VALUE'):
                return [u'{c.red}!!! Call ended by exception{c.reset}'.format(c=self.c)]

        value = self.highlighted(my_cheap_repr(arg))
        if event.comprehension_type:
            prefix = plain_prefix = u'Result: '
        else:
            plain_prefix = u'<<< {description} value from {func}: '.format(
                description='Yield' if opname == 'YIELD_VALUE' else 'Return',
                func=event.code_qualname(),
            )
            prefix = u'{c.green}{}{c.reset}'.format(
                plain_prefix,
                c=self.c,
            )
        return indented_lines(prefix, value, plain_prefix=plain_prefix)

    def get_statement_start_lines(self, event, this_statement, last_statement):
        result = []
        if (
                event.event != 'call' and
                this_statement and last_statement and
                this_statement != last_statement and
                this_statement.lineno != event.line_no and
                not isinstance(this_statement, try_statement)
        ):
            original_line_no = event.line_no
            for n in range(this_statement.lineno, original_line_no):
                event.line_no = n
                result.append(self.format_event(event))
            event.line_no = original_line_no
        return result

    def format_variables(self, event, last_statement):
        if last_statement:
            last_source_line = event.source.lines[last_statement.lineno - 1]
            dots = (get_leading_spaces(last_source_line)
                    .replace(' ', '.')
                    .replace('\t', '....'))
        else:
            dots = ''
            last_source_line = ''
        lines = []
        for var in event.variables:
            if ('{} = {}'.format(*var) != last_source_line.strip()
                    and not (
                            isinstance(last_statement, ast.FunctionDef)
                            and not last_statement.decorator_list
                            and var[0] == last_statement.name
                    )
            ):
                lines += self.format_variable(var, dots, event.comprehension_type)
        return lines

    def format_start(self, event):
        if event.comprehension_type:
            return ['{type}:'.format(type=event.comprehension_type)]
        else:
            if event.event == 'enter':
                description = 'Enter with block in'
            else:
                assert event.event == 'call'
                if event.frame_info.is_generator:
                    if event.opname == 'YIELD_VALUE':
                        description = 'Re-enter generator'
                    else:
                        description = 'Start generator'
                else:
                    description = 'Call to'
            return [
                u'{c.cyan}>>> {description} {name} in File "{filename}", line {lineno}{c.reset}'.format(
                    name=event.code_qualname(),
                    filename=_get_filename(event),
                    lineno=event.line_no,
                    c=self.c,
                    description=description,
                )]

    def format_executing_node_exception(self, event):
        try:
            assert not NO_ASTTOKENS
            call = Source.executing(event.frame).node
            if not isinstance(call, ast.Call):
                return []

            if any(
                    getattr(call, attr, None)
                    for attr in 'args keywords starargs kwargs'.split()
            ):
                args_source = '...'
            else:
                args_source = ''

            source = '{func}({args})'.format(
                func=with_needed_parentheses(event.source.get_text_with_indentation(call.func)),
                args=args_source,
            )
            plain_prefix = '!!! When calling: '
            prefix = '{c.red}{}{c.reset}'.format(plain_prefix, c=self.c)
            return indented_lines(
                prefix,
                source,
                plain_prefix=plain_prefix
            )
        except Exception:
            return []

    def columns_string(self, event):
        column_strings = []
        for column in self.columns:
            string = six.text_type(column(event))
            width = self.column_widths[column] = max(
                self.column_widths[column],
                len(string),
            )
            column_strings.append(string.ljust(width))
        return u' '.join(column_strings)

    def format_event(self, entry):
        return u'{c.grey}{line_no:4}{c.reset} | {source_line}'.format(
            source_line=self.highlighted_source_line(entry),
            c=self.c,
            **entry.__dict__
        )

    def format_variable(self, entry, dots, is_comprehension):
        name, value = entry
        if name.startswith('.') and name[1:].isdigit():
            description = 'Iterating over'
        elif is_comprehension:
            description = 'Values of {name}:'.format(name=name)
        else:
            description = '{name} ='.format(name=name)
        prefix = u'......{dots} {description} '.format(
            description=description,
            dots=dots,
        )
        return indented_lines(prefix, self.highlighted(value))

    def format_line_only(self, event):
        return self.format_lines(event, [self.format_event(event)])

    def format_log(self, event):
        return self.format_lines(event, ['LOG:'])

    def format_log_value(self, event, source, value, depth):
        prefix = u'....{} '.format(depth * 4 * '.')
        plain_source_lines = indented_lines(prefix, source)
        highl_source_lines = indented_lines(prefix, self.highlighted(source))

        plain_prefix = plain_source_lines[-1] + ' = '
        highl_prefix = highl_source_lines[-1] + ' = '

        lines = highl_source_lines[:-1] + indented_lines(
            highl_prefix,
            value,
            plain_prefix=plain_prefix
        )
        return self.format_lines(event, lines)

    def format_lines(self, event, lines):
        prefix = self.full_prefix(event)
        return ''.join([
            (
                    prefix
                    + line
                    + u'\n'
            )
            for line in lines
        ])

def get_leading_spaces(s):
    return s[:len(s) - len(s.lstrip())]

def _get_filename(event):
    # return "/path/to/example.py"
    return event.code.co_filename

class NoColors(object):
    def __getattr__(self, item):
        return ''

class Colors(object):
    grey = '\x1b[90m'
    red = '\x1b[31m\x1b[1m'
    green = '\x1b[32m\x1b[1m'
    cyan = '\x1b[36m\x1b[1m'
    reset = '\x1b[0m'

def indented_lines(prefix, string, plain_prefix=None):
    lines = six.text_type(string).splitlines() or ['']
    return [prefix + lines[0]] + [
        ' ' * len(plain_prefix or prefix) + line
        for line in lines[1:]
    ]
<commit_msg>Make default style readable on both light and dark backgrounds<commit_after>import ast
import opcode
import threading
import traceback
from collections import defaultdict
from datetime import datetime
from textwrap import dedent

import executing
import six
from pygments import highlight
from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.lexers.python import Python3Lexer, PythonLexer
from pygments.styles.monokai import MonokaiStyle
from six import PY3

from snoop.utils import ensure_tuple, short_filename, with_needed_parentheses, my_cheap_repr, \
    NO_ASTTOKENS, optional_numeric_label, try_statement, FormattedValue, ArgDefaultDict, lru_cache

class StatementsDict(dict):
    def __init__(self, source):
        super(StatementsDict, self).__init__()
        self.source = source

    def __missing__(self, key):
        statements = self.source.statements_at_line(key)
        if len(statements) == 1:
            result = list(statements)[0]
        else:
            result = None
        self[key] = result
        return result

class Source(executing.Source):
    def __init__(self, *args, **kwargs):
        super(Source, self).__init__(*args, **kwargs)
        if self.tree:
            self.lines = self.text.splitlines()
            self.highlighted = ArgDefaultDict(
                lambda style: raw_highlight(self.text, style).splitlines()
            )
        else:
            self.lines = defaultdict(lambda: u'SOURCE IS UNAVAILABLE')
            self.highlighted = defaultdict(lambda: self.lines)
        self.statements = StatementsDict(self)
        self.nodes = []
        if self.tree:
            self.tree._depth = 0
            for node in ast.walk(self.tree):
                node._tree_index = len(self.nodes)
                self.nodes.append(node)
                for child in ast.iter_child_nodes(node):
                    child._depth = node._depth + 1

    def get_text_with_indentation(self, node):
        result = self.asttokens().get_text(node)

        if not result:
            if isinstance(node, FormattedValue):
                fvals = [
                    n for n in node.parent.values
                    if isinstance(n, FormattedValue)
                ]
                return '<f-string value{}>'.format(
                    optional_numeric_label(
                        fvals.index(node),
                        fvals,
                    )
                )
            else:
                return "<unknown>"

        if '\n' in result:
            result = ' ' * node.first_token.start[1] + result
            result = dedent(result)
        else:
            result = result.strip()
        return result

if PY3:
    lexer = Python3Lexer()
else:
    lexer = PythonLexer()

class ForceWhiteTerminal256Formatter(Terminal256Formatter):
    def _closest_color(self, r, g, b):
        result = super(ForceWhiteTerminal256Formatter, self)._closest_color(r, g, b)
        if result == 15:
            result += 6 ** 3
        return result

class NeutralMonokaiStyle(MonokaiStyle):
    """
    Monokai style that is somewhat readable on both dark and light backgrounds
    by replacing 'white' styles with no style so that the terminal can automatically
    use the appropriate foreground color.
    """
    styles = {
        k: '' if v == '#f8f8f2' else v
        for k, v in MonokaiStyle.styles.items()
    }

formatters = ArgDefaultDict(lambda style: ForceWhiteTerminal256Formatter(style=style))

def raw_highlight(code, style):
    return highlight(code, lexer, formatters[style])

cached_highlight = lru_cache(maxsize=1024)(raw_highlight)

class Event(object):
    def __init__(self, frame_info, event, arg, depth, line_no=None):
        self.frame_info = frame_info
        self.frame = frame = frame_info.frame
        self.source = frame_info.source
        self.last_line_no = frame_info.last_line_no
        self.comprehension_type = frame_info.comprehension_type

        self.event = event
        self.arg = arg
        self.depth = depth

        self.variables = []
        if line_no is None:
            line_no = frame.f_lineno
        self.line_no = line_no
        self.code = frame.f_code

        if self.event == 'call' and self.source_line.lstrip().startswith('@'):
            # If a function decorator is found, skip lines until an actual
            # function definition is found.
            while True:
                self.line_no += 1
                try:
                    if self.source_line.lstrip().startswith('def'):
                        break
                except IndexError:
                    self.line_no = self.frame.lineno
                    break

    @property
    def source_line(self):
        return self.source.lines[self.line_no - 1]

    def code_qualname(self):
        return self.source.code_qualname(self.code)

    @property
    def opname(self):
        code_byte = self.code.co_code[self.frame.f_lasti]
        if not isinstance(code_byte, int):
            code_byte = ord(code_byte)
        return opcode.opname[code_byte]

class DefaultFormatter(object):
    datetime_format = None

    def __init__(self, prefix, columns, color):
        prefix = six.text_type(prefix)
        if prefix and prefix == prefix.rstrip():
            prefix += ' '
        self.prefix = prefix
        self.columns = [
            column if callable(column) else
            getattr(self, '{}_column'.format(column))
            for column in ensure_tuple(columns, split=True)
        ]
        self.column_widths = dict.fromkeys(self.columns, 0)
        if color is True:
            color = NeutralMonokaiStyle
        if color:
            self.c = Colors

            def highlighted(code):
                return cached_highlight(code, color)

            def highlighted_source_line(event):
                return event.source.highlighted[color][event.line_no - 1]
        else:
            self.c = NoColors()

            def highlighted(code):
                return code

            def highlighted_source_line(event):
                return event.source_line

        self.highlighted = highlighted
        self.highlighted_source_line = highlighted_source_line

    def thread_column(self, _event):
        return threading.current_thread().name

    def thread_ident_column(self, _event):
        return threading.current_thread().ident

    def time_column(self, _event):
        datetime_format = self.datetime_format or '%H:%M:%S.%f'
        result = datetime.now().strftime(datetime_format)
        if self.datetime_format is None:
            result = result[:-4]
        return result

    def file_column(self, event):
        return short_filename(event.code)

    def full_file_column(self, event):
        return _get_filename(event)

    def function_column(self, event):
        return event.code.co_name

    def function_qualname_column(self, event):
        return event.code_qualname()

    def full_prefix(self, event):
        return u'{c.grey}{self.prefix}{indent}{columns} {c.reset}'.format(
            c=self.c,
            self=self,
            indent=event.depth * u'    ',
            columns=self.columns_string(event),
        )

    def format(self, event):
        # type: (Event) -> str
        lines = []

        if event.event in ('call', 'enter'):
            lines += self.format_start(event)

        statements = event.source.statements
        this_statement = statements[event.line_no]
        last_statement = statements[event.last_line_no]
        statement_start_lines = self.get_statement_start_lines(event, this_statement, last_statement)

        lines += self.format_variables(event, last_statement)

        if event.event == 'return':
            lines += self.format_return(event)
        elif event.event == 'exception':
            lines += self.format_exception(event)
        elif event.event == 'enter':
            pass
        elif event.event == 'exit':
            lines += [u'{c.green}<<< Exit with block in {func}{c.reset}'.format(
                c=self.c,
                func=event.code_qualname(),
            )]
        else:
            if not (event.comprehension_type and event.event == 'line'):
                lines += statement_start_lines + [self.format_event(event)]

        return self.format_lines(event, lines)

    def format_exception(self, event):
        lines = []
        exception_string = ''.join(traceback.format_exception_only(*event.arg[:2]))
        lines += [
            u'{c.red}!!! {line}{c.reset}'.format(
                c=self.c,
                line=line,
            )
            for line in exception_string.splitlines()
        ]
        lines += self.format_executing_node_exception(event)
        return lines

    def format_return(self, event):
        # If a call ends due to an exception, we still get a 'return' event
        # with arg = None. This seems to be the only way to tell the difference
        # https://stackoverflow.com/a/12800909/2482744
        opname = event.opname
        arg = event.arg
        if arg is None:
            if opname == 'END_FINALLY':
                if event.frame_info.had_exception:
                    return [u'{c.red}??? Call either returned None or ended by exception{c.reset}'
                                .format(c=self.c)]
            elif opname not in ('RETURN_VALUE', 'YIELD_VALUE'):
                return [u'{c.red}!!! Call ended by exception{c.reset}'.format(c=self.c)]

        value = self.highlighted(my_cheap_repr(arg))
        if event.comprehension_type:
            prefix = plain_prefix = u'Result: '
        else:
            plain_prefix = u'<<< {description} value from {func}: '.format(
                description='Yield' if opname == 'YIELD_VALUE' else 'Return',
                func=event.code_qualname(),
            )
            prefix = u'{c.green}{}{c.reset}'.format(
                plain_prefix,
                c=self.c,
            )
        return indented_lines(prefix, value, plain_prefix=plain_prefix)

    def get_statement_start_lines(self, event, this_statement, last_statement):
        result = []
        if (
                event.event != 'call' and
                this_statement and last_statement and
                this_statement != last_statement and
                this_statement.lineno != event.line_no and
                not isinstance(this_statement, try_statement)
        ):
            original_line_no = event.line_no
            for n in range(this_statement.lineno, original_line_no):
                event.line_no = n
                result.append(self.format_event(event))
            event.line_no = original_line_no
        return result

    def format_variables(self, event, last_statement):
        if last_statement:
            last_source_line = event.source.lines[last_statement.lineno - 1]
            dots = (get_leading_spaces(last_source_line)
                    .replace(' ', '.')
                    .replace('\t', '....'))
        else:
            dots = ''
            last_source_line = ''
        lines = []
        for var in event.variables:
            if ('{} = {}'.format(*var) != last_source_line.strip()
                    and not (
                            isinstance(last_statement, ast.FunctionDef)
                            and not last_statement.decorator_list
                            and var[0] == last_statement.name
                    )
            ):
                lines += self.format_variable(var, dots, event.comprehension_type)
        return lines

    def format_start(self, event):
        if event.comprehension_type:
            return ['{type}:'.format(type=event.comprehension_type)]
        else:
            if event.event == 'enter':
                description = 'Enter with block in'
            else:
                assert event.event == 'call'
                if event.frame_info.is_generator:
                    if event.opname == 'YIELD_VALUE':
                        description = 'Re-enter generator'
                    else:
                        description = 'Start generator'
                else:
                    description = 'Call to'
            return [
                u'{c.cyan}>>> {description} {name} in File "{filename}", line {lineno}{c.reset}'.format(
                    name=event.code_qualname(),
                    filename=_get_filename(event),
                    lineno=event.line_no,
                    c=self.c,
                    description=description,
                )]

    def format_executing_node_exception(self, event):
        try:
            assert not NO_ASTTOKENS
            call = Source.executing(event.frame).node
            if not isinstance(call, ast.Call):
                return []

            if any(
                    getattr(call, attr, None)
                    for attr in 'args keywords starargs kwargs'.split()
            ):
                args_source = '...'
            else:
                args_source = ''

            source = '{func}({args})'.format(
                func=with_needed_parentheses(event.source.get_text_with_indentation(call.func)),
                args=args_source,
            )
            plain_prefix = '!!! When calling: '
            prefix = '{c.red}{}{c.reset}'.format(plain_prefix, c=self.c)
            return indented_lines(
                prefix,
                source,
                plain_prefix=plain_prefix
            )
        except Exception:
            return []

    def columns_string(self, event):
        column_strings = []
        for column in self.columns:
            string = six.text_type(column(event))
            width = self.column_widths[column] = max(
                self.column_widths[column],
                len(string),
            )
            column_strings.append(string.ljust(width))
        return u' '.join(column_strings)

    def format_event(self, entry):
        return u'{c.grey}{line_no:4}{c.reset} | {source_line}'.format(
            source_line=self.highlighted_source_line(entry),
            c=self.c,
            **entry.__dict__
        )

    def format_variable(self, entry, dots, is_comprehension):
        name, value = entry
        if name.startswith('.') and name[1:].isdigit():
            description = 'Iterating over'
        elif is_comprehension:
            description = 'Values of {name}:'.format(name=name)
        else:
            description = '{name} ='.format(name=name)
        prefix = u'......{dots} {description} '.format(
            description=description,
            dots=dots,
        )
        return indented_lines(prefix, self.highlighted(value))

    def format_line_only(self, event):
        return self.format_lines(event, [self.format_event(event)])

    def format_log(self, event):
        return self.format_lines(event, ['LOG:'])

    def format_log_value(self, event, source, value, depth):
        prefix = u'....{} '.format(depth * 4 * '.')
        plain_source_lines = indented_lines(prefix, source)
        highl_source_lines = indented_lines(prefix, self.highlighted(source))

        plain_prefix = plain_source_lines[-1] + ' = '
        highl_prefix = highl_source_lines[-1] + ' = '

        lines = highl_source_lines[:-1] + indented_lines(
            highl_prefix,
            value,
            plain_prefix=plain_prefix
        )
        return self.format_lines(event, lines)

    def format_lines(self, event, lines):
        prefix = self.full_prefix(event)
        return ''.join([
            (
                    prefix
                    + line
                    + u'\n'
            )
            for line in lines
        ])

def get_leading_spaces(s):
    return s[:len(s) - len(s.lstrip())]

def _get_filename(event):
    # return "/path/to/example.py"
    return event.code.co_filename

class NoColors(object):
    def __getattr__(self, item):
        return ''

class Colors(object):
    grey = '\x1b[90m'
    red = '\x1b[31m\x1b[1m'
    green = '\x1b[32m\x1b[1m'
    cyan = '\x1b[36m\x1b[1m'
    reset = '\x1b[0m'

def indented_lines(prefix, string, plain_prefix=None):
    lines = six.text_type(string).splitlines() or ['']
    return [prefix + lines[0]] + [
        ' ' * len(plain_prefix or prefix) + line
        for line in lines[1:]
    ]
<|endoftext|>


<commit_before>from django.test import TestCase

# Create your tests here.
<commit_msg>tests for profile.<commit_after>from django.test import TestCase, Client, RequestFactory
from django.contrib.auth.models import User
from profile_cfn.models import ProfileCfn
from profile_cfn.views import ProfileCfnView
import factory

class UserFactory(factory.django.DjangoModelFactory):
    """User factory for testing."""

    class Meta:
        """Model meta."""

        model = User

    username = factory.Sequence(lambda n: "User{}".format(n))
    email = factory.LazyAttribute(
        lambda x: "{}@imager.com".format(x.username.replace(" ", ""))
    )

class BackendTests(TestCase):
    """User/profile backend test runner."""

    def setUp(self):
        """Set up for backend test runner."""
        self.users = [UserFactory.create() for i in range(5)]

    def test_profile_is_made_when_user_is_saved(self):
        """Test profile created when user is registered."""
        self.assertTrue(ProfileCfn.objects.count() == 5)

    def test_profile_is_associated_with_users(self):
        """Test profile has correct user instance."""
        profile = ProfileCfn.objects.first()
        username = self.users[0].username
        self.assertTrue(profile.user.username == username)

class FrontendTests(TestCase):
    """User/profile frontend test runner."""

    def setUp(self):
        """Set up for frontend fest runner."""
        self.client = Client()
        self.request = RequestFactory()

    def add_user(self):
        """Make a user and return his profile."""
        user = UserFactory.create()
        user.username = 'test_user'
        user.set_password('testpassword')
        user.save()
        return user

    def test_profile_view_returns_status_ok(self):
        """Test profile view returns status code 200."""
        test_user = self.add_user()
        import pdb; pdb.set_trace()
        self.client.force_login(test_user)
        response = self.client.get("/profile")
        self.assertTrue(response.status_code == 200)
<|endoftext|>


<commit_before>from soapy.wsdl.element import Element

""" Models are pythonic objects representing the XML tags in a WSDL, but outside of the Types tag """

class Service(Element):
    """ Simplified, native Python representation of Service definitions in WSDL
    Provides information on child (port) objects by name and service tag attribute information """

    @property
    def ports(self) -> tuple:
        try:
            return self.__ports
        except AttributeError:
            self.log("Initializing list of ports defined for service {0}".format(self.name), 5)
            ports = list()
            for port in self.bs_element('port', recursive=False):
                ports.append(Port(port, self.parent))
            self.__ports = tuple(ports)
            return self.__ports

class PortType(Element):
    """ Simplified, native Python representation of portType definition in WSDL """

    @property
    def operations(self) -> tuple:
        try:
            return self.__operations
        except AttributeError:
            self.log("Initializing operations for portType {0} from wsdl".format(self.name), 5)
            operations = list()
            for operation in self.bs_element('operation', recursive=False):
                operations.append(Operation(operation, self.parent))
            self.__operations = tuple(operations)
            return self.__operations

    @property
    def methods(self) -> tuple:
        return self.operations

class Binding(Element):
    """ Simplified, native python representation of a binding definition
    Also provides enforcement that the style of the binding is document
    as this library does not (currently) support other styles """

    def __init__(self, bsElement, parent):
        super().__init__(bsElement, parent)
        # Validate that the binding style is 'document'
        soapBinding = bsElement('binding', recursive=False)[0]
        if not soapBinding['style'] == "document":
            self.log("Binding style not set to document. SoaPy can't handle non-document styles", 0)
            raise TypeError("Binding style not set to document. SoaPy can't handle non-document styles")

    @property
    def type(self) -> PortType:
        try:
            return self.__type
        except AttributeError:
            self.log("Initializing portType from binding {0}".format(self.name), 5)
            (ns, name) = self.bs_element['type'].split(":")
            self.__type = PortType.from_name(name, self.parent)
            return self.__type

    def get_soap_action(self, op_name) -> str:

        """ Given the name of an operation, return the soap action """

        operations = self.bs_element('operation', recursive=False)
        for operation in operations:
            if operation['name'] == op_name:
                soap_op = operation('operation')[0]
                try:
                    return soap_op['soapAction']
                except KeyError:
                    self.log("Binding operation does not contain a soapAction element", 2)
                    return None

        self.log("Could not find matching operation, {0} in binding {1}".format(
            op_name, self.name), 2)

class Port(Element):
    """ Simplified, native Python representation of ports as defined within services """

    @property
    def binding(self) -> Binding:
        try:
            return self.__binding
        except AttributeError:
            self.log("Initializing binding attribute for port {0}".format(self.name), 5)
            binding = self.bs_element['binding']
            (ns, name) = binding.split(':')
            self.__binding = Binding.from_name(name, self.parent)
            return self.__binding

    @property
    def location(self) -> str:
        try:
            return self.__location
        except AttributeError:
            self.log("Initializing location of Port based on address element", 5)
            self.__location = self.bs_element('address', recursive=False)[0]['location']
            self.log("Initialized location to {0}".format(self.__location), 4)
            return self.__location

    @location.setter
    def location(self, location):
        self.__location = location

class Message(Element):
    @property
    def parts(self) -> tuple:
        try:
            return self.__parts
        except AttributeError:
            self.log("Initializing parts for message {0}".format(self.name), 5)
            parts = list()
            for part in self.bs_element('part', recursive=False):
                parts.append(Part(part, self.parent))
            self.__parts = tuple(parts)
            return self.__parts

class Part(Element):
    @property
    def type(self) -> Element:
        try:
            return self.__type
        except AttributeError:
            self.__type = self.parent.find_type_by_name(self.bs_element['element'])
            return self.__type

    @property
    def element(self) -> Element:
        return self.type

    @property
    def ns(self) -> str:
        return self.bs_element['element'].split(':')[0]

class Operation(Element):
    """ Simplified, native Python representation of an operation definition in portType element group"""

    @property
    def input(self) -> Message:
        try:
            return self.__input
        except AttributeError:
            name = self.bs_element('input')[0].get('message').split(":")[1]
            self.__input = Message.from_name(name, self.parent)
            return self.__input

    @property
    def output(self) -> Message:
        try:
            return self.__output
        except AttributeError:
            name = self.bs_element('output')[0].get('message').split(":")[1]
            self.__output = Message.from_name(name, self.parent)
            return self.__output

    @property
    def faults(self) -> tuple:
        try:
            return self.__faults
        except AttributeError:
            search_results = self.bs_element('fault')
            if len(search_results) > 0:
                faults = list()
                for each in search_results:
                    faults.append(Message.from_name(each.get("message").split(":")[1], self.parent))
                self.__faults = tuple(faults)
                return self.__faults
            else:
                self.log("Operation has no fault message specified", 2)
                self.__faults = tuple()
                return self.__faults
<commit_msg>Added support for version 1.2 WSDLs Added support for specifying authentication method to client request Added proper default document style support to binding element<commit_after>from soapy.wsdl.element import Element

""" Models are pythonic objects representing the XML tags in a WSDL, but outside of the Types tag """

class Service(Element):
    """ Simplified, native Python representation of Service definitions in WSDL
    Provides information on child (port) objects by name and service tag attribute information """

    @property
    def ports(self) -> tuple:
        try:
            return self.__ports
        except AttributeError:
            self.log("Initializing list of ports defined for service {0}".format(self.name), 5)
            ports = list()
            for port in self.bs_element('port', recursive=False):
                ports.append(Port(port, self.parent))
            self.__ports = tuple(ports)
            return self.__ports

class PortType(Element):
    """ Simplified, native Python representation of portType definition in WSDL """

    @property
    def operations(self) -> tuple:
        try:
            return self.__operations
        except AttributeError:
            self.log("Initializing operations for portType {0} from wsdl".format(self.name), 5)
            operations = list()
            for operation in self.bs_element('operation', recursive=False):
                operations.append(Operation(operation, self.parent))
            self.__operations = tuple(operations)
            return self.__operations

    @property
    def methods(self) -> tuple:
        return self.operations

class Binding(Element):
    """ Simplified, native python representation of a binding definition
    Also provides enforcement that the style of the binding is document
    as this library does not (currently) support other styles """

    def __init__(self, bsElement, parent):
        super().__init__(bsElement, parent)
        # Validate that the binding style is 'document'
        soapBinding = bsElement('binding', recursive=False)[0]
        if not soapBinding.get('style', "document") == "document":
            self.log("Binding style not set to document. SoaPy can't handle non-document styles", 0)
            raise TypeError("Binding style not set to document. SoaPy can't handle non-document styles")

    @property
    def type(self) -> PortType:
        try:
            return self.__type
        except AttributeError:
            self.log("Initializing portType from binding {0}".format(self.name), 5)
            (ns, name) = self.bs_element['type'].split(":")
            self.__type = PortType.from_name(name, self.parent)
            return self.__type

    def get_soap_action(self, op_name) -> str:

        """ Given the name of an operation, return the soap action """

        operations = self.bs_element('operation', recursive=False)
        for operation in operations:
            if operation['name'] == op_name:
                soap_op = operation('operation')[0]
                try:
                    return soap_op['soapAction']
                except KeyError:
                    self.log("Binding operation does not contain a soapAction element", 2)
                    return None

        self.log("Could not find matching operation, {0} in binding {1}".format(
            op_name, self.name), 2)

class Port(Element):
    """ Simplified, native Python representation of ports as defined within services """

    @property
    def binding(self) -> Binding:
        try:
            return self.__binding
        except AttributeError:
            self.log("Initializing binding attribute for port {0}".format(self.name), 5)
            binding = self.bs_element['binding']
            (ns, name) = binding.split(':')
            self.__binding = Binding.from_name(name, self.parent)
            return self.__binding

    @property
    def location(self) -> str:
        try:
            return self.__location
        except AttributeError:
            self.log("Initializing location of Port based on address element", 5)
            self.__location = self.bs_element('address', recursive=False)[0]['location']
            self.log("Initialized location to {0}".format(self.__location), 4)
            return self.__location

    @location.setter
    def location(self, location):
        self.__location = location

class Message(Element):
    @property
    def parts(self) -> tuple:
        try:
            return self.__parts
        except AttributeError:
            self.log("Initializing parts for message {0}".format(self.name), 5)
            parts = list()
            for part in self.bs_element('part', recursive=False):
                parts.append(Part(part, self.parent))
            self.__parts = tuple(parts)
            return self.__parts

class Part(Element):
    @property
    def type(self) -> Element:
        try:
            return self.__type
        except AttributeError:
            self.__type = self.parent.find_type_by_name(self.bs_element['element'])
            return self.__type

    @property
    def element(self) -> Element:
        return self.type

    @property
    def ns(self) -> str:
        return self.bs_element['element'].split(':')[0]

class Operation(Element):
    """ Simplified, native Python representation of an operation definition in portType element group"""

    @property
    def input(self) -> Message:
        try:
            return self.__input
        except AttributeError:
            name = self.bs_element('input')[0].get('message').split(":")[1]
            self.__input = Message.from_name(name, self.parent)
            return self.__input

    @property
    def output(self) -> Message:
        try:
            return self.__output
        except AttributeError:
            name = self.bs_element('output')[0].get('message').split(":")[1]
            self.__output = Message.from_name(name, self.parent)
            return self.__output

    @property
    def faults(self) -> tuple:
        try:
            return self.__faults
        except AttributeError:
            search_results = self.bs_element('fault')
            if len(search_results) > 0:
                faults = list()
                for each in search_results:
                    faults.append(Message.from_name(each.get("message").split(":")[1], self.parent))
                self.__faults = tuple(faults)
                return self.__faults
            else:
                self.log("Operation has no fault message specified", 2)
                self.__faults = tuple()
                return self.__faults
<|endoftext|>


<commit_before>#!/usr/bin/env python

import psycopg2
import cgi
import sys
import unittest

def pil_logic(s):
    """Convert the CGI pil value into something we can query

    Args:
      s (str): The CGI variable wanted

    Returns:
      list of PILs to send to the databae"""
    if s == '':
        return []
    s = s.upper()
    pils = []
    if s.find(",") == -1:
        pils.append(s)
    else:
        pils = s.split(",")

    res = []
    for pil in pils:
        if pil[:3] == "WAR":
            for q in ['FLS', 'FFS', 'AWW', 'TOR', 'SVR', 'FFW', 'SVS',
                      'LSR', 'SPS', 'WSW', 'FFA', 'WCN']:
                res.append("%s%s" % (q, pil[3:6]))
        else:
            res.append("%6.6s" % (pil.strip() + '      ', ))
    return res

def main():
    """Process the request"""
    # Attempt to keep the file from downloading and just displaying in chrome
    sys.stdout.write("X-Content-Type-Options: nosniff\n")
    sys.stdout.write("Content-type: text/plain\n\n")
    form = cgi.FieldStorage()
    pils = pil_logic(form.getfirst('pil', ''))
    limit = int(form.getfirst('limit', 1))
    center = form.getfirst('center', '')[:4]
    sdate = form.getfirst('sdate', '')[:10]
    edate = form.getfirst('edate', '')[:10]
    fmt = form.getfirst('fmt', 'text')
    if len(pils) == 0:
        sys.stdout.write("ERROR: No pil specified...")
        return
    centerlimit = '' if center == '' else (" and source = '%s' " % (center, ))
    timelimit = ''
    if sdate != '' and edate != '':
        timelimit = (" and entered >= '%s' and entered < '%s' "
                     ) % (sdate, edate)

    if pils[0][:3] == 'MTR':
        access = psycopg2.connect(database='iem', host='iemdb', user='nobody')
        cursor = access.cursor()
        sql = """
            SELECT raw from current_log c JOIN stations t
            on (t.iemid = c.iemid)
            WHERE raw != '' and id = '%s' ORDER by valid DESC LIMIT %s
            """ % (pils[0][3:].strip(), limit)
        cursor.execute(sql)
        for row in cursor:
            sys.stdout.write("\001\n")
            sys.stdout.write(row[0].replace("\r\r\n", "\n"))
            sys.stdout.write("\n\003")
        if cursor.rowcount == 0:
            sys.stdout.write("ERROR: METAR lookup for %s failed" % (
                                                pils[0][3:].strip(), ))
        return

    try:
        mydb = psycopg2.connect(database='afos', host='iemdb', user='nobody')
    except:
        print 'Error Connecting to Database, please try again!'
        return

    cursor = mydb.cursor()

    if len(pils) == 1:
        pillimit = " pil = '%s' " % (pils[0], )
    else:
        pillimit = " pil in %s" % (tuple(pils), )

    # Do optimized query first, see if we can get our limit right away
    sql = """
        SELECT data from products WHERE %s
        and entered > now() - '2 days'::interval %s %s
        ORDER by entered DESC LIMIT %s""" % (pillimit, centerlimit,
                                             timelimit, limit)

    cursor.execute(sql)
    if cursor.rowcount != limit:
        sql = """
            SELECT data from products WHERE %s %s %s
            ORDER by entered DESC LIMIT %s """ % (pillimit, centerlimit,
                                                  timelimit, limit)
        cursor.execute(sql)

    for row in cursor:
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\001\n")
        # Remove control characters from the product as we are including
        # them manually here...
        sys.stdout.write((row[0]).replace(
            "\003", "").replace("\001\r\r\n", "").replace("\r\r\n", "\n"))
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\n\003\n")

    if cursor.rowcount == 0:
        print("ERROR: Could not Find: %s" % (",".join(pils), ))

if __name__ == '__main__':
    main()

class tester(unittest.TestCase):

    def test_pil_logic(self):
        """Make sure our pil logic works! """
        res = pil_logic("AFDDMX")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0], 'AFDDMX')
        res = pil_logic("WAREWX")
        self.assertEquals(len(res), 12)
        res = pil_logic("STOIA,AFDDMX")
        self.assertEquals(res[0], 'STOIA ')
        self.assertEquals(res[1], 'AFDDMX')
<commit_msg>fix up some html things with AFOS retrieve, whack-a-mole<commit_after>#!/usr/bin/env python

import psycopg2
import cgi
import sys
import unittest

def pil_logic(s):
    """Convert the CGI pil value into something we can query

    Args:
      s (str): The CGI variable wanted

    Returns:
      list of PILs to send to the databae"""
    if s == '':
        return []
    s = s.upper()
    pils = []
    if s.find(",") == -1:
        pils.append(s)
    else:
        pils = s.split(",")

    res = []
    for pil in pils:
        if pil[:3] == "WAR":
            for q in ['FLS', 'FFS', 'AWW', 'TOR', 'SVR', 'FFW', 'SVS',
                      'LSR', 'SPS', 'WSW', 'FFA', 'WCN']:
                res.append("%s%s" % (q, pil[3:6]))
        else:
            res.append("%6.6s" % (pil.strip() + '      ', ))
    return res

def main():
    """Process the request"""
    # Attempt to keep the file from downloading and just displaying in chrome
    form = cgi.FieldStorage()
    pils = pil_logic(form.getfirst('pil', ''))
    limit = int(form.getfirst('limit', 1))
    center = form.getfirst('center', '')[:4]
    sdate = form.getfirst('sdate', '')[:10]
    edate = form.getfirst('edate', '')[:10]
    fmt = form.getfirst('fmt', 'text')
    sys.stdout.write("X-Content-Type-Options: nosniff\n")
    if fmt == 'text':
        sys.stdout.write("Content-type: text/plain\n\n")
    elif fmt == 'html':
        sys.stdout.write("Content-type: text/html\n\n")
    if len(pils) == 0:
        sys.stdout.write("ERROR: No pil specified...")
        return
    centerlimit = '' if center == '' else (" and source = '%s' " % (center, ))
    timelimit = ''
    if sdate != '' and edate != '':
        timelimit = (" and entered >= '%s' and entered < '%s' "
                     ) % (sdate, edate)

    if pils[0][:3] == 'MTR':
        access = psycopg2.connect(database='iem', host='iemdb', user='nobody')
        cursor = access.cursor()
        sql = """
            SELECT raw from current_log c JOIN stations t
            on (t.iemid = c.iemid)
            WHERE raw != '' and id = '%s' ORDER by valid DESC LIMIT %s
            """ % (pils[0][3:].strip(), limit)
        cursor.execute(sql)
        for row in cursor:
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\001\n")
            sys.stdout.write(row[0].replace("\r\r\n", "\n"))
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\003\n")
        if cursor.rowcount == 0:
            sys.stdout.write("ERROR: METAR lookup for %s failed" % (
                                                pils[0][3:].strip(), ))
        return

    try:
        mydb = psycopg2.connect(database='afos', host='iemdb', user='nobody')
    except:
        print 'Error Connecting to Database, please try again!'
        return

    cursor = mydb.cursor()

    if len(pils) == 1:
        pillimit = " pil = '%s' " % (pils[0], )
    else:
        pillimit = " pil in %s" % (tuple(pils), )

    # Do optimized query first, see if we can get our limit right away
    sql = """
        SELECT data from products WHERE %s
        and entered > now() - '31 days'::interval %s %s
        ORDER by entered DESC LIMIT %s""" % (pillimit, centerlimit,
                                             timelimit, limit)

    cursor.execute(sql)
    if cursor.rowcount != limit:
        sql = """
            SELECT data from products WHERE %s %s %s
            ORDER by entered DESC LIMIT %s """ % (pillimit, centerlimit,
                                                  timelimit, limit)
        cursor.execute(sql)

    for row in cursor:
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\001\n")
        # Remove control characters from the product as we are including
        # them manually here...
        sys.stdout.write((row[0]).replace(
            "\003", "").replace("\001\r\r\n", "").replace("\r\r\n", "\n"))
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\n\003\n")

    if cursor.rowcount == 0:
        print("ERROR: Could not Find: %s" % (",".join(pils), ))

if __name__ == '__main__':
    main()

class tester(unittest.TestCase):

    def test_pil_logic(self):
        """Make sure our pil logic works! """
        res = pil_logic("AFDDMX")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0], 'AFDDMX')
        res = pil_logic("WAREWX")
        self.assertEquals(len(res), 12)
        res = pil_logic("STOIA,AFDDMX")
        self.assertEquals(res[0], 'STOIA ')
        self.assertEquals(res[1], 'AFDDMX')
<|endoftext|>


<commit_before>#solarnmf_main_ts.py

#Will Barnes
#31 March 2015

#Import needed modules
import solarnmf_functions as snf
import solarnmf_plot_routines as spr

#Set some initial parameters
P = 10 #simulated number of sources
Q = 10 #guessed number of sources
angle = 45 #rotation angle
nx,ny = 100,100 #dimensions for simulation

#Read in and format the time series
results = snf.make_t_matrix("simulation",format="timeseries",nx=nx,ny=ny,p=P,angle=anglefilename='/home/wtb2/Desktop/gaussian_test.dat')

#Get the dimensions of the T matrix
#ny,nx = results['T'].shape

#Initialize the U, V, and A matrices
uva_initial = snf.initialize_uva(nx,ny,Q,10,10,results['T'])

#Start the minimizer
min_results = snf.minimize_div(uva_initial['u'],uva_initial['v'],results['T'],uva_initial['A'],500,1.0e-5)

#Show the initial and final matrices side-by-side
spr.plot_mat_obsVpred(results['T'],min_results['A'])

#Show the initial and final 1d time series curves
spr.plot_ts_obsVpred(results['x'],min_results['A'],angle=-angle)

#Show the constituents of the time series on top of the original vector
spr.plot_ts_reconstruction(results['x'],min_results['u'],min_results['v'],angle=-angle)

#Show the convergence of the distance metric
spr.plot_convergence(min_results['div'])
<commit_msg>Fix: needed keyword in make_t_matrix function call<commit_after>#solarnmf_main_ts.py

#Will Barnes
#31 March 2015

#Import needed modules
import solarnmf_functions as snf
import solarnmf_plot_routines as spr

#Set some initial parameters
P = 10 #simulated number of sources
Q = 10 #guessed number of sources
angle = 45 #rotation angle
nx,ny = 100,100 #dimensions for simulation

#Read in and format the time series
results = snf.make_t_matrix("simulation",format="timeseries",nx=nx,ny=ny,p=P,angle=angle,filename='/home/wtb2/Desktop/gaussian_test.dat')

#Get the dimensions of the T matrix
#ny,nx = results['T'].shape

#Initialize the U, V, and A matrices
uva_initial = snf.initialize_uva(nx,ny,Q,10,10,results['T'])

#Start the minimizer
min_results = snf.minimize_div(uva_initial['u'],uva_initial['v'],results['T'],uva_initial['A'],500,1.0e-5)

#Show the initial and final matrices side-by-side
spr.plot_mat_obsVpred(results['T'],min_results['A'])

#Show the initial and final 1d time series curves
spr.plot_ts_obsVpred(results['x'],min_results['A'],angle=-angle)

#Show the constituents of the time series on top of the original vector
spr.plot_ts_reconstruction(results['x'],min_results['u'],min_results['v'],angle=-angle)

#Show the convergence of the distance metric
spr.plot_convergence(min_results['div'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import psycopg2
import cgi
import sys
import unittest

def pil_logic(s):
    """Convert the CGI pil value into something we can query

    Args:
      s (str): The CGI variable wanted

    Returns:
      list of PILs to send to the databae"""
    if s == '':
        return []
    s = s.upper()
    pils = []
    if s.find(",") == -1:
        pils.append(s)
    else:
        pils = s.split(",")

    res = []
    for pil in pils:
        if pil[:3] == "WAR":
            for q in ['FLS', 'FFS', 'AWW', 'TOR', 'SVR', 'FFW', 'SVS',
                      'LSR', 'SPS', 'WSW', 'FFA', 'WCN']:
                res.append("%s%s" % (q, pil[3:6]))
        else:
            res.append("%6.6s" % (pil.strip() + '      ', ))
    return res

def main():
    """Process the request"""
    # Attempt to keep the file from downloading and just displaying in chrome
    form = cgi.FieldStorage()
    pils = pil_logic(form.getfirst('pil', ''))
    limit = int(form.getfirst('limit', 1))
    center = form.getfirst('center', '')[:4]
    sdate = form.getfirst('sdate', '')[:10]
    edate = form.getfirst('edate', '')[:10]
    fmt = form.getfirst('fmt', 'text')
    sys.stdout.write("X-Content-Type-Options: nosniff\n")
    if fmt == 'text':
        sys.stdout.write("Content-type: text/plain\n\n")
    elif fmt == 'html':
        sys.stdout.write("Content-type: text/html\n\n")
    if len(pils) == 0:
        sys.stdout.write("ERROR: No pil specified...")
        return
    centerlimit = '' if center == '' else (" and source = '%s' " % (center, ))
    timelimit = ''
    if sdate != '' and edate != '':
        timelimit = (" and entered >= '%s' and entered < '%s' "
                     ) % (sdate, edate)

    if pils[0][:3] == 'MTR':
        access = psycopg2.connect(database='iem', host='iemdb', user='nobody')
        cursor = access.cursor()
        sql = """
            SELECT raw from current_log c JOIN stations t
            on (t.iemid = c.iemid)
            WHERE raw != '' and id = '%s' ORDER by valid DESC LIMIT %s
            """ % (pils[0][3:].strip(), limit)
        cursor.execute(sql)
        for row in cursor:
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\001\n")
            sys.stdout.write(row[0].replace("\r\r\n", "\n"))
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\003\n")
        if cursor.rowcount == 0:
            sys.stdout.write("ERROR: METAR lookup for %s failed" % (
                                                pils[0][3:].strip(), ))
        return

    try:
        mydb = psycopg2.connect(database='afos', host='iemdb', user='nobody')
    except:
        print 'Error Connecting to Database, please try again!'
        return

    cursor = mydb.cursor()

    if len(pils) == 1:
        pillimit = " pil = '%s' " % (pils[0], )
    else:
        pillimit = " pil in %s" % (tuple(pils), )

    # Do optimized query first, see if we can get our limit right away
    sql = """
        SELECT data from products WHERE %s
        and entered > now() - '31 days'::interval %s %s
        ORDER by entered DESC LIMIT %s""" % (pillimit, centerlimit,
                                             timelimit, limit)

    cursor.execute(sql)
    if cursor.rowcount != limit:
        sql = """
            SELECT data from products WHERE %s %s %s
            ORDER by entered DESC LIMIT %s """ % (pillimit, centerlimit,
                                                  timelimit, limit)
        cursor.execute(sql)

    for row in cursor:
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\001\n")
        # Remove control characters from the product as we are including
        # them manually here...
        sys.stdout.write((row[0]).replace(
            "\003", "").replace("\001\r\r\n", "").replace("\r\r\n", "\n"))
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\n\003\n")

    if cursor.rowcount == 0:
        print("ERROR: Could not Find: %s" % (",".join(pils), ))

if __name__ == '__main__':
    main()

class tester(unittest.TestCase):

    def test_pil_logic(self):
        """Make sure our pil logic works! """
        res = pil_logic("AFDDMX")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0], 'AFDDMX')
        res = pil_logic("WAREWX")
        self.assertEquals(len(res), 12)
        res = pil_logic("STOIA,AFDDMX")
        self.assertEquals(res[0], 'STOIA ')
        self.assertEquals(res[1], 'AFDDMX')
<commit_msg>non-func pylint while reviewing<commit_after>#!/usr/bin/env python
"""give me some AFOS data please"""
from __future__ import print_function
import cgi
import sys
import unittest

import psycopg2

def pil_logic(s):
    """Convert the CGI pil value into something we can query

    Args:
      s (str): The CGI variable wanted

    Returns:
      list of PILs to send to the databae"""
    if s == '':
        return []
    s = s.upper()
    pils = []
    if s.find(",") == -1:
        pils.append(s)
    else:
        pils = s.split(",")

    res = []
    for pil in pils:
        if pil[:3] == "WAR":
            for q in ['FLS', 'FFS', 'AWW', 'TOR', 'SVR', 'FFW', 'SVS',
                      'LSR', 'SPS', 'WSW', 'FFA', 'WCN']:
                res.append("%s%s" % (q, pil[3:6]))
        else:
            res.append("%6.6s" % (pil.strip() + '      ', ))
    return res

def main():
    """Process the request"""
    # Attempt to keep the file from downloading and just displaying in chrome
    form = cgi.FieldStorage()
    pils = pil_logic(form.getfirst('pil', ''))
    limit = int(form.getfirst('limit', 1))
    center = form.getfirst('center', '')[:4]
    sdate = form.getfirst('sdate', '')[:10]
    edate = form.getfirst('edate', '')[:10]
    fmt = form.getfirst('fmt', 'text')
    sys.stdout.write("X-Content-Type-Options: nosniff\n")
    if fmt == 'text':
        sys.stdout.write("Content-type: text/plain\n\n")
    elif fmt == 'html':
        sys.stdout.write("Content-type: text/html\n\n")
    if pils:
        sys.stdout.write("ERROR: No pil specified...")
        return
    centerlimit = '' if center == '' else (" and source = '%s' " % (center, ))
    timelimit = ''
    if sdate != '' and edate != '':
        timelimit = (" and entered >= '%s' and entered < '%s' "
                     ) % (sdate, edate)

    if pils[0][:3] == 'MTR':
        access = psycopg2.connect(database='iem', host='iemdb', user='nobody')
        cursor = access.cursor()
        sql = """
            SELECT raw from current_log c JOIN stations t
            on (t.iemid = c.iemid)
            WHERE raw != '' and id = '%s' ORDER by valid DESC LIMIT %s
            """ % (pils[0][3:].strip(), limit)
        cursor.execute(sql)
        for row in cursor:
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\001\n")
            sys.stdout.write(row[0].replace("\r\r\n", "\n"))
            if fmt == 'html':
                sys.stdout.write("
\n")
            else:
                sys.stdout.write("\003\n")
        if cursor.rowcount == 0:
            sys.stdout.write("ERROR: METAR lookup for %s failed" % (
                                                pils[0][3:].strip(), ))
        return

    try:
        mydb = psycopg2.connect(database='afos', host='iemdb', user='nobody')
    except Exception as exp:
        print('Error Connecting to Database, please try again!')
        return

    cursor = mydb.cursor()

    if len(pils) == 1:
        pillimit = " pil = '%s' " % (pils[0], )
    else:
        pillimit = " pil in %s" % (tuple(pils), )

    # Do optimized query first, see if we can get our limit right away
    sql = """
        SELECT data from products WHERE %s
        and entered > now() - '31 days'::interval %s %s
        ORDER by entered DESC LIMIT %s""" % (pillimit, centerlimit,
                                             timelimit, limit)

    cursor.execute(sql)
    if cursor.rowcount != limit:
        sql = """
            SELECT data from products WHERE %s %s %s
            ORDER by entered DESC LIMIT %s """ % (pillimit, centerlimit,
                                                  timelimit, limit)
        cursor.execute(sql)

    for row in cursor:
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\001\n")
        # Remove control characters from the product as we are including
        # them manually here...
        sys.stdout.write((row[0]).replace(
            "\003", "").replace("\001\r\r\n", "").replace("\r\r\n", "\n"))
        if fmt == 'html':
            sys.stdout.write("
\n")
        else:
            sys.stdout.write("\n\003\n")

    if cursor.rowcount == 0:
        print("ERROR: Could not Find: %s" % (",".join(pils), ))

if __name__ == '__main__':
    main()

class TestRetrieve(unittest.TestCase):
    """some tests"""

    def test_pil_logic(self):
        """Make sure our pil logic works! """
        res = pil_logic("AFDDMX")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0], 'AFDDMX')
        res = pil_logic("WAREWX")
        self.assertEquals(len(res), 12)
        res = pil_logic("STOIA,AFDDMX")
        self.assertEquals(res[0], 'STOIA ')
        self.assertEquals(res[1], 'AFDDMX')
<|endoftext|>


<commit_before>"""
Classes that introduce disruption in clusters.
"""

import inspect
import random
import time

from .data_path import get_data_path

NODETOOL_CMD_TIMEOUT = 240

class Nemesis(object):

    def __init__(self, cluster, termination_event):
        self.cluster = cluster
        self.node_to_operate = None
        self.set_node_to_operate()
        self.termination_event = termination_event

    def set_node_to_operate(self):
        self.node_to_operate = random.choice(self.cluster.nodes)
        print('Node to operate: {}'.format(self.node_to_operate))

    def run(self, interval=30, termination_event=None):
        interval *= 60
        while True:
            time.sleep(interval)
            self.disrupt()
            if termination_event is not None:
                if self.termination_event.isSet():
                    self.termination_event = None
                    break
            self.set_node_to_operate()

    def disrupt(self):
        raise NotImplementedError('Derived classes must implement disrupt()')

    def disrupt_nodetool_drain(self):
        self.node_to_operate.remoter.run('nodetool -h localhost drain',
                                         timeout=NODETOOL_CMD_TIMEOUT)
        self.node_to_operate.instance.stop()
        time.sleep(60)
        self.node_to_operate.instance.start()
        self.node_to_operate.wait_for_init()

    def disrupt_nodetool_decommission(self):
        node_to_operate_ip = self.node_to_operate.instance.private_ip_address
        self.node_to_operate.remoter.run('nodetool --host localhost '
                                         'decommission',
                                         timeout=NODETOOL_CMD_TIMEOUT)
        verification_node = random.choice(self.cluster.nodes)
        while verification_node == self.node_to_operate:
            verification_node = random.choice(self.cluster.nodes)

        node_info_list = self.cluster.get_node_info_list(verification_node)
        private_ips = [node_info['ip'] for node_info in node_info_list]
        error_msg = ('Node that was decommissioned {} still in the cluster. '
                     'Cluster status info: {}'.format(self.node_to_operate,
                                                      node_info_list))
        assert node_to_operate_ip not in private_ips, error_msg
        self.cluster.nodes.remove(self.node_to_operate)
        self.node_to_operate.instance.terminate()

    def disrupt_stop_start(self):
        self.node_to_operate.instance.stop()
        time.sleep(60)
        self.node_to_operate.instance.start()

    def disrupt_kill_scylla_daemon(self):
        self.node_to_operate.remoter.run('sudo killall -9 scylla')

    def _destroy_data(self):
        # Send the script used to corrupt the DB
        break_scylla = get_data_path('break_scylla.sh')
        self.node_to_operate.remoter.send_files(break_scylla,
                                                "/tmp/break_scylla.sh")

        # corrupt the DB
        self.node_to_operate.remoter.run('chmod +x /tmp/break_scylla.sh')
        self.node_to_operate.remoter.run('/tmp/break_scylla.sh')

        # lennart's systemd will restart scylla let him a bit of time
        self.disrupt_kill_scylla_daemon()
        time.sleep(60)

    def disrupt_destroy_data_then_repair(self):
        self._destroy_data()
        # try to save the node
        self.repair_nodetool_repair()
        time.sleep(60)

    def disrupt_destroy_data_then_rebuild(self):
        self._destroy_data()
        # try to save the node
        self.repair_nodetool_rebuild()
        time.sleep(60)

    def call_random_disrupt_method(self):
        disrupt_methods = [attr[1] for attr in inspect.getmembers(self) if
                           attr[0].startswith('disrupt_') and
                           callable(attr[1])]
        disrupt_method = random.choice(disrupt_methods)
        disrupt_method()

    def repair_nodetool_repair(self):
        self.node_to_operate.remoter.run('nodetool -h localhost repair',
                                         timeout=NODETOOL_CMD_TIMEOUT)

    def repair_nodetool_rebuild(self):
        for node in self.cluster.nodes:
            node.remoter.run_parallel('nodetool -h localhost rebuild',
                                      timeout=NODETOOL_CMD_TIMEOUT)

class StopStartMonkey(Nemesis):

    def disrupt(self):
        self.disrupt_stop_start()

class DrainerMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_nodetool_drain()

class CorruptThenRepairMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_destroy_data_then_repair()

class CorruptThenRebuildMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_destroy_data_then_rebuild()

class DecommissionMonkey(Nemesis):

    def disrupt(self):
        self.disrupt_nodetool_decommission()

class ChaosMonkey(Nemesis):

    def disrupt(self):
        self.call_random_disrupt_method()
<commit_msg>sdcm.nemesis: Add better logging to nemesis operations<commit_after>"""
Classes that introduce disruption in clusters.
"""

import inspect
import random
import time

from .data_path import get_data_path

NODETOOL_CMD_TIMEOUT = 240

class Nemesis(object):

    def __init__(self, cluster, termination_event):
        self.cluster = cluster
        self.node_to_operate = None
        self.set_node_to_operate()
        self.termination_event = termination_event

    def set_node_to_operate(self):
        self.node_to_operate = random.choice(self.cluster.nodes)
        print('Node to operate: {}'.format(self.node_to_operate))

    def run(self, interval=30, termination_event=None):
        interval *= 60
        while True:
            time.sleep(interval)
            self.disrupt()
            if termination_event is not None:
                if self.termination_event.isSet():
                    self.termination_event = None
                    break
            self.set_node_to_operate()

    def __str__(self):
        return str(self.__class__)

    def disrupt(self):
        raise NotImplementedError('Derived classes must implement disrupt()')

    def disrupt_nodetool_drain(self):
        print('{}: Drain {} then restart it'.format(self, self.node_to_operate))
        self.node_to_operate.remoter.run('nodetool -h localhost drain',
                                         timeout=NODETOOL_CMD_TIMEOUT)
        self.node_to_operate.instance.stop()
        time.sleep(60)
        self.node_to_operate.instance.start()
        self.node_to_operate.wait_for_init()

    def disrupt_nodetool_decommission(self):
        print('{}: Decomission {}'.format(self, self.node_to_operate))
        node_to_operate_ip = self.node_to_operate.instance.private_ip_address
        self.node_to_operate.remoter.run('nodetool --host localhost '
                                         'decommission',
                                         timeout=NODETOOL_CMD_TIMEOUT)
        verification_node = random.choice(self.cluster.nodes)
        while verification_node == self.node_to_operate:
            verification_node = random.choice(self.cluster.nodes)

        node_info_list = self.cluster.get_node_info_list(verification_node)
        private_ips = [node_info['ip'] for node_info in node_info_list]
        error_msg = ('Node that was decommissioned {} still in the cluster. '
                     'Cluster status info: {}'.format(self.node_to_operate,
                                                      node_info_list))
        assert node_to_operate_ip not in private_ips, error_msg
        self.cluster.nodes.remove(self.node_to_operate)
        self.node_to_operate.instance.terminate()

    def disrupt_stop_start(self):
        print('{}: Stop {} then restart it'.format(self, self.node_to_operate))
        self.node_to_operate.instance.stop()
        time.sleep(60)
        self.node_to_operate.instance.start()

    def disrupt_kill_scylla_daemon(self):
        print('{}: Kill all scylla processes in {}'.format(self, self.node_to_operate))
        self.node_to_operate.remoter.run('sudo killall -9 scylla')

    def _destroy_data(self):
        # Send the script used to corrupt the DB
        break_scylla = get_data_path('break_scylla.sh')
        self.node_to_operate.remoter.send_files(break_scylla,
                                                "/tmp/break_scylla.sh")

        # corrupt the DB
        self.node_to_operate.remoter.run('chmod +x /tmp/break_scylla.sh')
        self.node_to_operate.remoter.run('/tmp/break_scylla.sh')

        # lennart's systemd will restart scylla let him a bit of time
        self.disrupt_kill_scylla_daemon()
        time.sleep(60)

    def disrupt_destroy_data_then_repair(self):
        print('{}: Destroy user data in {}, then run nodetool '
              'repair'.format(self, self.node_to_operate))
        self._destroy_data()
        # try to save the node
        self.repair_nodetool_repair()
        time.sleep(60)

    def disrupt_destroy_data_then_rebuild(self):
        print('{}: Destroy user data in {}, then run nodetool '
              'rebuild'.format(self, self.node_to_operate))
        self._destroy_data()
        # try to save the node
        self.repair_nodetool_rebuild()
        time.sleep(60)

    def call_random_disrupt_method(self):
        disrupt_methods = [attr[1] for attr in inspect.getmembers(self) if
                           attr[0].startswith('disrupt_') and
                           callable(attr[1])]
        disrupt_method = random.choice(disrupt_methods)
        disrupt_method()

    def repair_nodetool_repair(self):
        self.node_to_operate.remoter.run('nodetool -h localhost repair',
                                         timeout=NODETOOL_CMD_TIMEOUT)

    def repair_nodetool_rebuild(self):
        for node in self.cluster.nodes:
            node.remoter.run_parallel('nodetool -h localhost rebuild',
                                      timeout=NODETOOL_CMD_TIMEOUT)

class StopStartMonkey(Nemesis):

    def disrupt(self):
        self.disrupt_stop_start()

class DrainerMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_nodetool_drain()

class CorruptThenRepairMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_destroy_data_then_repair()

class CorruptThenRebuildMonkey(Nemesis):

    def run(self, interval=30, termination_event=None):
        interval *= 60
        time.sleep(interval)
        self.disrupt()

    def disrupt(self):
        self.disrupt_destroy_data_then_rebuild()

class DecommissionMonkey(Nemesis):

    def disrupt(self):
        self.disrupt_nodetool_decommission()

class ChaosMonkey(Nemesis):

    def disrupt(self):
        self.call_random_disrupt_method()
<|endoftext|>


<commit_before>"""Receives documents from the oplog worker threads and indexes them into the backend.

This file is a document manager for the Solr search engine, but the intent is that this file can be used as an example to add on different backends.
"""
#!/usr/env/python
import sys

from pysolr import Solr
from threading import Timer
from util import verify_url, retry_until_ok

class DocManager():
    """The DocManager class creates a connection to the backend engine and adds/removes documents, and in the case of rollback, searches for them.

    The reason for storing id/doc pairs as opposed to doc's is so that multiple
    updates to the same doc reflect the most up to date version as opposed to
    multiple, slightly different versions of a doc.
    """

    def __init__(self, url, auto_commit = True, unique_key = '_id'):
        """Verify Solr URL and establish a connection.

        This method may vary from implementation to implementation, but it must
        verify the url to the backend and return None if that fails. It must
        also create the connection to the backend, and start a periodic
        committer if necessary. The Solr uniqueKey is set to '_id' in the sample
        schema, but this may be overridden by user defined configuration.
        """
        if verify_url(url) is False:
		    print 'Invalid Solr URL'
		    return None

        self.solr = Solr(url)
        self.unique_key = unique_key
        if auto_commit:
        	self.auto_commit()

    def upsert(self, doc):
        """Update or insert a document into Solr

        This method should call whatever add/insert/update method exists for the
        backend engine and add the document in there. The input will always be
        one mongo document, represented as a Python dictionary.
        """

        self.solr.add([doc], commit=False)

    def remove(self, doc):
        """Removes documents from Solr

        The input is a python dictionary that represents a mongo document.
        """
        self.solr.delete(id=str(doc[self.unique_key]), commit=False)

    def search(self, start_ts, end_ts):
        """Called to query Solr for documents in a time range.

        This method is only used by rollbacks to query all the documents in Solr
        within a certain timestamp window. The input will be two longs
        (converted from Bson timestamp) which specify the time range. The output
        should be an iterable set of documents.
        """
        query = '_ts: [%s TO %s]' % (start_ts, end_ts)

        return self.solr.search(query)

    def commit(self):
        """This function is used to force a commit.

        It is used only in the beginning of rollbacks and in test cases, and is
        not meant to be called in other circumstances. The body should commit
        all documents to the backend engine (like auto_commit), but not have any
        timers or run itself again (unlike auto_commit). In the event of too
        many Solr searchers, the commit is wrapped in a retry_until_ok to keep
        trying until the commit goes through.
        """
        retry_until_ok(self.solr.commit)

    def auto_commit(self):
        """Periodically commits to the Solr server.

        This function commits all changes to the Solr engine, and then starts a
        timer that calls this function again in one second. The reason for this
        function is to prevent overloading Solr from other searchers. This
        function may be modified based on the backend engine and how commits are
        handled, as timers may not be necessary in all instances.
        """
        self.solr.commit()
        Timer(1, self.solr_commit).start()

    def get_last_doc(self):
        """Returns the last document stored in the Solr engine.

        This method is used for rollbacks to establish the rollback window,
        which is the gap between the last document on a mongo shard and the last
        document in Solr.
        """
        #search everything, sort by descending timestamp, return 1 row
        result = self.solr.search('*:*', sort='_ts desc', rows=1)

        if len(result) == 0:
            return None

        return result.docs[0]

<commit_msg>cleaned up code for solr_doc_manager<commit_after>"""Receives documents from the oplog worker threads and indexes them into the backend.

This file is a document manager for the Solr search engine, but the intent is that this file can be used as an example to add on different backends. To extend this to other systems, simply implement the exact same class and replace the method definitions with API calls for the desired backend. Each method is detailed to describe the desired behavior.
"""
#!/usr/env/python
import sys

from pysolr import Solr
from threading import Timer
from util import verify_url, retry_until_ok

class DocManager():
    """The DocManager class creates a connection to the backend engine and adds/removes documents, and in the case of rollback, searches for them.

    The reason for storing id/doc pairs as opposed to doc's is so that multiple
    updates to the same doc reflect the most up to date version as opposed to
    multiple, slightly different versions of a doc.
    """

    def __init__(self, url, auto_commit = True, unique_key = '_id'):
        """Verify Solr URL and establish a connection.

        This method may vary from implementation to implementation, but it must
        verify the url to the backend and return None if that fails. It must
        also create the connection to the backend, and start a periodic
        committer if necessary. The Solr uniqueKey is set to '_id' in the sample
        schema, but this may be overridden by user defined configuration.
        """
        if verify_url(url) is False:
		    print 'Invalid Solr URL'
		    return None

        self.solr = Solr(url)
        self.unique_key = unique_key
        if auto_commit:
        	self.auto_commit()

    def upsert(self, doc):
        """Update or insert a document into Solr

        This method should call whatever add/insert/update method exists for the
        backend engine and add the document in there. The input will always be
        one mongo document, represented as a Python dictionary.
        """
        self.solr.add([doc], commit=False)

    def remove(self, doc):
        """Removes documents from Solr

        The input is a python dictionary that represents a mongo document.
        """
        self.solr.delete(id=str(doc[self.unique_key]), commit=False)

    def search(self, start_ts, end_ts):
        """Called to query Solr for documents in a time range.

        This method is only used by rollbacks to query all the documents in Solr
        within a certain timestamp window. The input will be two longs
        (converted from Bson timestamp) which specify the time range. The return
        value should be an iterable set of documents.
        """
        query = '_ts: [%s TO %s]' % (start_ts, end_ts)
        return self.solr.search(query)

    def commit(self):
        """This function is used to force a commit.

        It is used only in the beginning of rollbacks and in test cases, and is
        not meant to be called in other circumstances. The body should commit
        all documents to the backend engine (like auto_commit), but not have any
        timers or run itself again (unlike auto_commit). In the event of too
        many Solr searchers, the commit is wrapped in a retry_until_ok to keep
        trying until the commit goes through.
        """
        retry_until_ok(self.solr.commit)

    def auto_commit(self):
        """Periodically commits to the Solr server.

        This function commits all changes to the Solr engine, and then starts a
        timer that calls this function again in one second. The reason for this
        function is to prevent overloading Solr from other searchers. This
        function may be modified based on the backend engine and how commits are
        handled, as timers may not be necessary in all instances.
        """
        self.solr.commit()
        Timer(1, self.solr_commit).start()

    def get_last_doc(self):
        """Returns the last document stored in the Solr engine.

        This method is used for rollbacks to establish the rollback window,
        which is the gap between the last document on a mongo shard and the last
        document in Solr. If there are no documents, this functions returns
        None. Otherwise, it returns the first document.
        """
        #search everything, sort by descending timestamp, return 1 row
        result = self.solr.search('*:*', sort='_ts desc', rows=1)

        if len(result) == 0:
            return None

        return result.docs[0]

<|endoftext|>