<commit_before>import os
import sys
import traceback
import time
from threading import Thread

from gateway.observer import Event

sys.path.insert(0, '/opt/openmotics/python')

from platform_utils import System
System.import_eggs()

from toolbox import PluginIPCStream
from plugin_runtime import base
from plugin_runtime.utils import get_plugin_class, check_plugin, get_special_methods
from plugin_runtime.interfaces import has_interface
from plugin_runtime.web import WebInterfaceDispatcher

class PluginRuntime:

    def __init__(self, path):
        self._stopped = False
        self._path = path.rstrip('/')

        self._input_status_receivers = []
        self._output_status_receivers = []
        self._shutter_status_receivers = []
        self._event_receivers = []

        self._name = None
        self._version = None
        self._interfaces = []
        self._receivers = []
        self._exposes = []
        self._metric_definitions = []
        self._metric_collectors = []
        self._metric_receivers = []

        self._plugin = None

        self._webinterface = WebInterfaceDispatcher(IO._log)

    def _init_plugin(self):
        plugin_root = os.path.dirname(self._path)
        plugin_dir = os.path.basename(self._path)

        # Add the plugin and it's eggs to the python path
        sys.path.insert(0, plugin_root)
        for egg_file in os.listdir(self._path):
            if egg_file.endswith('.egg'):
                sys.path.append(os.path.join(self._path, egg_file))

        # Expose plugins.base to the plugin
        sys.modules['plugins'] = sys.modules['__main__']
        sys.modules["plugins.base"] = base

        # Instanciate the plugin class
        plugin_class = get_plugin_class(plugin_dir)
        check_plugin(plugin_class)

        # Set the name, version, interfaces
        self._name = plugin_class.name
        self._version = plugin_class.version
        self._interfaces = plugin_class.interfaces

        # Initialze the plugin
        self._plugin = plugin_class(self._webinterface, IO._log)

        # Set the receivers
        receiver_mapping = {'input_status': self._input_status_receivers,
                            'output_status': self._output_status_receivers,
                            'shutter_status': self._shutter_status_receivers,
                            'receive_events': self._event_receivers}

        for method_attribute, target in receiver_mapping.iteritems():
            for method in get_special_methods(self._plugin, method_attribute):
                target.append(method)

            if len(target) > 0:
                self._receivers.append(method_attribute)

        # Set the exposed methods
        for method in get_special_methods(self._plugin, 'om_expose'):
            self._exposes.append({'name': method.__name__,
                                  'auth': method.om_expose['auth'],
                                  'content_type': method.om_expose['content_type']})

        # Set the metric definitions
        if has_interface(plugin_class, 'metrics', '1.0'):
            if hasattr(plugin_class, 'metric_definitions'):
                self._metric_definitions = plugin_class.metric_definitions

        # Set the metric collectors
        for method in get_special_methods(self._plugin, 'om_metric_data'):
            self._metric_collectors.append({'name': method.__name__,
                                            'interval': method.om_metric_data['interval']})

        # Set the metric receivers
        for method in get_special_methods(self._plugin, 'om_metric_receive'):
            self._metric_receivers.append({'name': method.__name__,
                                           'source': method.om_metric_receive['source'],
                                           'metric_type': method.om_metric_receive['metric_type'],
                                           'interval': method.om_metric_receive['interval']})

    def _start_background_tasks(self):
        """ Start all background tasks. """
        tasks = get_special_methods(self._plugin, 'background_task')
        for task in tasks:
            thread = Thread(target=PluginRuntime._run_background_task, args=(task,))
            thread.name = 'Background thread ({0})'.format(task.__name__)
            thread.daemon = True
            thread.start()

    @staticmethod
    def _run_background_task(task):
        running = True
        while running:
            try:
                task()
                running = False  # Stop execution if the task returns without exception
            except Exception as exception:
                IO._log_exception('background task', exception)
                time.sleep(30)

    def process_stdin(self):
        while not self._stopped:
            command = IO._wait_and_read_command()
            if command is None:
                continue

            action = command['action']
            response = {'cid': command['cid'], 'action': action}
            try:
                ret = None
                if action == 'start':
                    ret = self._handle_start()
                elif action == 'stop':
                    ret = self._handle_stop()
                elif action == 'input_status':
                    ret = self._handle_input_status(command['event'])
                elif action == 'output_status':
                    ret = self._handle_output_status(command['status'])
                elif action == 'shutter_status':
                    ret = self._handle_shutter_status(command)
                elif action == 'receive_events':
                    ret = self._handle_receive_events(command['code'])
                elif action == 'get_metric_definitions':
                    ret = self._handle_get_metric_definitions()
                elif action == 'collect_metrics':
                    ret = self._handle_collect_metrics(command['name'])
                elif action == 'distribute_metric':
                    ret = self._handle_distribute_metric(command['name'], command['metric'])
                elif action == 'request':
                    ret = self._handle_request(command['method'], command['args'], command['kwargs'])
                elif action == 'remove_callback':
                    ret = self._handle_remove_callback()
                else:
                    raise RuntimeError('Unknown action: {0}'.format(action))

                if ret is not None:
                    response.update(ret)
            except Exception as exception:
                response['_exception'] = str(exception)
            IO._write(response)

    def _handle_start(self):
        """ Handles the start command. Cover exceptions manually to make sure as much metadata is returned as possible. """
        data = {}
        try:
            self._init_plugin()
            self._start_background_tasks()
        except Exception as exception:
            data['exception'] = str(exception)
        data.update({'name': self._name,
                     'version': self._version,
                     'receivers': self._receivers,
                     'exposes': self._exposes,
                     'interfaces': self._interfaces,
                     'metric_collectors': self._metric_collectors,
                     'metric_receivers': self._metric_receivers})
        return data

    def _handle_stop(self):
        def delayed_stop():
            time.sleep(2)
            os._exit(0)

        stop_thread = Thread(target=delayed_stop)
        stop_thread.daemon = True
        stop_thread.start()

        self._stopped = True

    def _handle_input_status(self, event_json):
        event = Event.deserialize(event_json)
        # get relevant event details
        input_id = event.data['id']
        status = event.data['status']
        for receiver in self._input_status_receivers:
            version = receiver.input_status.get('version', 1)
            if version == 1:
                # Backwards compatibility: only send rising edges of the input (no input releases)
                if status:
                    IO._with_catch('input status', receiver, [(input_id, None)])
            elif version == 2:
                # Version 2 will send ALL input status changes AND in a dict format
                data = {'input_id': input_id, 'status': status}
                IO._with_catch('input status', receiver, [data])
            else:
                error = NotImplementedError('Version {} is not supported for input status decorators'.format(version))
                IO._log_exception('input status', error)

    def _handle_output_status(self, status):
        for receiver in self._output_status_receivers:
            IO._with_catch('output status', receiver, [status])

    def _handle_shutter_status(self, status):
        for receiver in self._shutter_status_receivers:
            if receiver.shutter_status['add_detail']:
                IO._with_catch('shutter status', receiver, [status['status'], status['detail']])
            else:
                IO._with_catch('shutter status', receiver, [status['status']])

    def _handle_receive_events(self, code):
        for receiver in self._event_receivers:
            IO._with_catch('process event', receiver, [code])

    def _handle_get_metric_definitions(self):
        return {'metric_definitions': self._metric_definitions}

    def _handle_collect_metrics(self, name):
        metrics = []
        collect = getattr(self._plugin, name)
        try:
            metrics.extend(list(collect()))
        except Exception as exception:
            IO._log_exception('collect metrics', exception)
        return {'metrics': metrics}

    def _handle_distribute_metric(self, name, metric):
        receive = getattr(self._plugin, name)
        IO._with_catch('distribute metric', receive, [metric])

    def _handle_request(self, method, args, kwargs):
        func = getattr(self._plugin, method)
        try:
            return {'success': True, 'response': func(*args, **kwargs)}
        except Exception as exception:
            return {'success': False, 'exception': str(exception), 'stacktrace': traceback.format_exc()}

    def _handle_remove_callback(self):
        for method in get_special_methods(self._plugin, 'on_remove'):
            try:
                method()
            except Exception as exception:
                IO._log_exception('on remove', exception)

class IO(object):
    @staticmethod
    def _log(msg):
        IO._write({'cid': 0, 'action': 'logs', 'logs': str(msg)})

    @staticmethod
    def _log_exception(name, exception):
        IO._log('Exception ({0}) in {1}: {2}'.format(exception, name, traceback.format_exc()))

    @staticmethod
    def _with_catch(name, target, args):
        """ Logs Exceptions that happen in target(*args). """
        try:
            return target(*args)
        except Exception as exception:
            IO._log_exception(name, exception)

    @staticmethod
    def _wait_and_read_command():
        stream = PluginIPCStream()
        while True:
            try:
                line = sys.stdin.readline()
                if line is None:
                    line = ''
            except Exception:
                continue
            try:
                response = stream.feed(line)
                if response is not None:
                    return response
            except Exception as ex:
                IO._log('Exception in _wait_and_read_command: Could not decode stdin: {0}'.format(ex))

    @staticmethod
    def _write(msg):
        sys.stdout.write(PluginIPCStream.encode(msg))
        sys.stdout.flush()

if __name__ == '__main__':
    if len(sys.argv) < 3 or sys.argv[1] != 'start':
        sys.stderr.write('Usage: python {0} start <path>\n'.format(sys.argv[0]))
        sys.stderr.flush()
        sys.exit(1)

    def watch_parent():
        parent = os.getppid()
        # If the parent process gets kills, this process will be attached to init.
        # In that case the plugin should stop running.
        while True:
            if os.getppid() != parent:
                os._exit(1)
            time.sleep(1)

    # Keep an eye on our parent process
    watcher = Thread(target=watch_parent)
    watcher.daemon = True
    watcher.start()

    # Start the runtime
    try:
        runtime = PluginRuntime(path=sys.argv[2])
        runtime.process_stdin()
    except BaseException as ex:
        IO._log_exception('__main__', ex)
        os._exit(1)

    os._exit(0)
<commit_msg>Cherry picked #203 to fix plugin regression<commit_after>import os
import sys
import traceback
import time
from threading import Thread

sys.path.insert(0, '/opt/openmotics/python')

from platform_utils import System
System.import_eggs()

from toolbox import PluginIPCStream
from gateway.observer import Event
from plugin_runtime import base
from plugin_runtime.utils import get_plugin_class, check_plugin, get_special_methods
from plugin_runtime.interfaces import has_interface
from plugin_runtime.web import WebInterfaceDispatcher

class PluginRuntime:

    def __init__(self, path):
        self._stopped = False
        self._path = path.rstrip('/')

        self._input_status_receivers = []
        self._output_status_receivers = []
        self._shutter_status_receivers = []
        self._event_receivers = []

        self._name = None
        self._version = None
        self._interfaces = []
        self._receivers = []
        self._exposes = []
        self._metric_definitions = []
        self._metric_collectors = []
        self._metric_receivers = []

        self._plugin = None

        self._webinterface = WebInterfaceDispatcher(IO._log)

    def _init_plugin(self):
        plugin_root = os.path.dirname(self._path)
        plugin_dir = os.path.basename(self._path)

        # Add the plugin and it's eggs to the python path
        sys.path.insert(0, plugin_root)
        for egg_file in os.listdir(self._path):
            if egg_file.endswith('.egg'):
                sys.path.append(os.path.join(self._path, egg_file))

        # Expose plugins.base to the plugin
        sys.modules['plugins'] = sys.modules['__main__']
        sys.modules["plugins.base"] = base

        # Instanciate the plugin class
        plugin_class = get_plugin_class(plugin_dir)
        check_plugin(plugin_class)

        # Set the name, version, interfaces
        self._name = plugin_class.name
        self._version = plugin_class.version
        self._interfaces = plugin_class.interfaces

        # Initialze the plugin
        self._plugin = plugin_class(self._webinterface, IO._log)

        # Set the receivers
        receiver_mapping = {'input_status': self._input_status_receivers,
                            'output_status': self._output_status_receivers,
                            'shutter_status': self._shutter_status_receivers,
                            'receive_events': self._event_receivers}

        for method_attribute, target in receiver_mapping.iteritems():
            for method in get_special_methods(self._plugin, method_attribute):
                target.append(method)

            if len(target) > 0:
                self._receivers.append(method_attribute)

        # Set the exposed methods
        for method in get_special_methods(self._plugin, 'om_expose'):
            self._exposes.append({'name': method.__name__,
                                  'auth': method.om_expose['auth'],
                                  'content_type': method.om_expose['content_type']})

        # Set the metric definitions
        if has_interface(plugin_class, 'metrics', '1.0'):
            if hasattr(plugin_class, 'metric_definitions'):
                self._metric_definitions = plugin_class.metric_definitions

        # Set the metric collectors
        for method in get_special_methods(self._plugin, 'om_metric_data'):
            self._metric_collectors.append({'name': method.__name__,
                                            'interval': method.om_metric_data['interval']})

        # Set the metric receivers
        for method in get_special_methods(self._plugin, 'om_metric_receive'):
            self._metric_receivers.append({'name': method.__name__,
                                           'source': method.om_metric_receive['source'],
                                           'metric_type': method.om_metric_receive['metric_type'],
                                           'interval': method.om_metric_receive['interval']})

    def _start_background_tasks(self):
        """ Start all background tasks. """
        tasks = get_special_methods(self._plugin, 'background_task')
        for task in tasks:
            thread = Thread(target=PluginRuntime._run_background_task, args=(task,))
            thread.name = 'Background thread ({0})'.format(task.__name__)
            thread.daemon = True
            thread.start()

    @staticmethod
    def _run_background_task(task):
        running = True
        while running:
            try:
                task()
                running = False  # Stop execution if the task returns without exception
            except Exception as exception:
                IO._log_exception('background task', exception)
                time.sleep(30)

    def process_stdin(self):
        while not self._stopped:
            command = IO._wait_and_read_command()
            if command is None:
                continue

            action = command['action']
            response = {'cid': command['cid'], 'action': action}
            try:
                ret = None
                if action == 'start':
                    ret = self._handle_start()
                elif action == 'stop':
                    ret = self._handle_stop()
                elif action == 'input_status':
                    ret = self._handle_input_status(command['event'])
                elif action == 'output_status':
                    ret = self._handle_output_status(command['status'])
                elif action == 'shutter_status':
                    ret = self._handle_shutter_status(command)
                elif action == 'receive_events':
                    ret = self._handle_receive_events(command['code'])
                elif action == 'get_metric_definitions':
                    ret = self._handle_get_metric_definitions()
                elif action == 'collect_metrics':
                    ret = self._handle_collect_metrics(command['name'])
                elif action == 'distribute_metric':
                    ret = self._handle_distribute_metric(command['name'], command['metric'])
                elif action == 'request':
                    ret = self._handle_request(command['method'], command['args'], command['kwargs'])
                elif action == 'remove_callback':
                    ret = self._handle_remove_callback()
                else:
                    raise RuntimeError('Unknown action: {0}'.format(action))

                if ret is not None:
                    response.update(ret)
            except Exception as exception:
                response['_exception'] = str(exception)
            IO._write(response)

    def _handle_start(self):
        """ Handles the start command. Cover exceptions manually to make sure as much metadata is returned as possible. """
        data = {}
        try:
            self._init_plugin()
            self._start_background_tasks()
        except Exception as exception:
            data['exception'] = str(exception)
        data.update({'name': self._name,
                     'version': self._version,
                     'receivers': self._receivers,
                     'exposes': self._exposes,
                     'interfaces': self._interfaces,
                     'metric_collectors': self._metric_collectors,
                     'metric_receivers': self._metric_receivers})
        return data

    def _handle_stop(self):
        def delayed_stop():
            time.sleep(2)
            os._exit(0)

        stop_thread = Thread(target=delayed_stop)
        stop_thread.daemon = True
        stop_thread.start()

        self._stopped = True

    def _handle_input_status(self, event_json):
        event = Event.deserialize(event_json)
        # get relevant event details
        input_id = event.data['id']
        status = event.data['status']
        for receiver in self._input_status_receivers:
            version = receiver.input_status.get('version', 1)
            if version == 1:
                # Backwards compatibility: only send rising edges of the input (no input releases)
                if status:
                    IO._with_catch('input status', receiver, [(input_id, None)])
            elif version == 2:
                # Version 2 will send ALL input status changes AND in a dict format
                data = {'input_id': input_id, 'status': status}
                IO._with_catch('input status', receiver, [data])
            else:
                error = NotImplementedError('Version {} is not supported for input status decorators'.format(version))
                IO._log_exception('input status', error)

    def _handle_output_status(self, status):
        for receiver in self._output_status_receivers:
            IO._with_catch('output status', receiver, [status])

    def _handle_shutter_status(self, status):
        for receiver in self._shutter_status_receivers:
            if receiver.shutter_status['add_detail']:
                IO._with_catch('shutter status', receiver, [status['status'], status['detail']])
            else:
                IO._with_catch('shutter status', receiver, [status['status']])

    def _handle_receive_events(self, code):
        for receiver in self._event_receivers:
            IO._with_catch('process event', receiver, [code])

    def _handle_get_metric_definitions(self):
        return {'metric_definitions': self._metric_definitions}

    def _handle_collect_metrics(self, name):
        metrics = []
        collect = getattr(self._plugin, name)
        try:
            metrics.extend(list(collect()))
        except Exception as exception:
            IO._log_exception('collect metrics', exception)
        return {'metrics': metrics}

    def _handle_distribute_metric(self, name, metric):
        receive = getattr(self._plugin, name)
        IO._with_catch('distribute metric', receive, [metric])

    def _handle_request(self, method, args, kwargs):
        func = getattr(self._plugin, method)
        try:
            return {'success': True, 'response': func(*args, **kwargs)}
        except Exception as exception:
            return {'success': False, 'exception': str(exception), 'stacktrace': traceback.format_exc()}

    def _handle_remove_callback(self):
        for method in get_special_methods(self._plugin, 'on_remove'):
            try:
                method()
            except Exception as exception:
                IO._log_exception('on remove', exception)

class IO(object):
    @staticmethod
    def _log(msg):
        IO._write({'cid': 0, 'action': 'logs', 'logs': str(msg)})

    @staticmethod
    def _log_exception(name, exception):
        IO._log('Exception ({0}) in {1}: {2}'.format(exception, name, traceback.format_exc()))

    @staticmethod
    def _with_catch(name, target, args):
        """ Logs Exceptions that happen in target(*args). """
        try:
            return target(*args)
        except Exception as exception:
            IO._log_exception(name, exception)

    @staticmethod
    def _wait_and_read_command():
        stream = PluginIPCStream()
        while True:
            try:
                line = sys.stdin.readline()
                if line is None:
                    line = ''
            except Exception:
                continue
            try:
                response = stream.feed(line)
                if response is not None:
                    return response
            except Exception as ex:
                IO._log('Exception in _wait_and_read_command: Could not decode stdin: {0}'.format(ex))

    @staticmethod
    def _write(msg):
        sys.stdout.write(PluginIPCStream.encode(msg))
        sys.stdout.flush()

if __name__ == '__main__':
    if len(sys.argv) < 3 or sys.argv[1] != 'start':
        sys.stderr.write('Usage: python {0} start <path>\n'.format(sys.argv[0]))
        sys.stderr.flush()
        sys.exit(1)

    def watch_parent():
        parent = os.getppid()
        # If the parent process gets kills, this process will be attached to init.
        # In that case the plugin should stop running.
        while True:
            if os.getppid() != parent:
                os._exit(1)
            time.sleep(1)

    # Keep an eye on our parent process
    watcher = Thread(target=watch_parent)
    watcher.daemon = True
    watcher.start()

    # Start the runtime
    try:
        runtime = PluginRuntime(path=sys.argv[2])
        runtime.process_stdin()
    except BaseException as ex:
        IO._log_exception('__main__', ex)
        os._exit(1)

    os._exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import random

import ephem

from nodes import Node

class Random(Node):
    char = "H"
    args = 1
    results = 1

    def random_choice(self, inp:Node.indexable):
        """Choose one in a list randomly"""
        return [random.choice(inp)]

    def randint(self, inp:int):
        """Random number between 0 and inp inclusive"""
        return random.randint(0,inp)

    def get_next_new_moon(self, time: Node.clock):
        """Gets the date of the next full moon"""
        new_time = datetime.datetime(*time.time_obj[:7])

        return ephem.next_full_moon(new_time)
<commit_msg>Fix get next new moon<commit_after>#!/usr/bin/env python

import datetime
import random

import ephem

from nodes import Node

class Random(Node):
    char = "H"
    args = 1
    results = 1

    def random_choice(self, inp:Node.indexable):
        """Choose one in a list randomly"""
        return [random.choice(inp)]

    def randint(self, inp:int):
        """Random number between 0 and inp inclusive"""
        return random.randint(0,inp)

    def get_next_new_moon(self, time: Node.clock):
        """Gets the date of the next new moon"""
        new_time = datetime.datetime(*time.time_obj[:7])

        return ephem.next_new_moon(new_time)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`noncegenerator` --- Nonce bullshit generator
==================================================
"""

__docformat__ = "restructuredtext en"

from string import punctuation
from collections import OrderedDict, deque
from random import choice, shuffle, uniform
from math import ceil

from Skype4Py.enums import cmsReceived, cmeEmoted

from plugin import Plugin

VOWELS = u"аеёиоуыэюя"

SUBSTITUTES = {
    u"а": u"хуя",
    u"е": u"хуе",
    u"ё": u"хуё",
    u"и": u"хуи",
    u"о": u"хуё",
    u"у": u"хую",
    u"ы": u"хуи",
    u"э": u"хуэ",
    u"ю": u"хую",
    u"я": u"хуя",
}

EXTRA_WORDS = [
    # u"тян",
    # u"кун",
    u"лал",
    u"лойс",
    u"азаза",
    u"нарм",
    u"бахает",
    u"бахнет",
    u"бахнул",
    u"лах",
    u"бомбанул",
    u"бамбан",
    u"пукан",
    u"пердак",
    u"выйгр",
    u"сасай",
    u"сасал",
    u"саси",
    u"сасн",
    u"здело",
]

TEMPLATES_SIMPLE = [
    u"{nonce_word}",
    u"{word}-{nonce_word}",
]

TEMPLATES_COMPOSITE = [
    u"{word}-{nonce_word}",
    u"{nonce_word}",
    u"{word}, блядь, {nonce_word}",
    u"{nonce_word}, блядь",
    u"{nonce_word}, нахуй",
    u"{word}-{nonce_word}, блядь",
    u"{word}-{nonce_word}, нахуй",
]

TEMPLATES_NONCE_ONLY = [
    u"{nonce_word}",
]

def _translate(s, characters, translate_to=u" "):
    translation_table = dict((ord(char), translate_to) for char in characters)
    return s.translate(translation_table)

def find_first_vowel_index(word):
    """
    >>> assert find_first_vowel_index("учёт".decode("utf8")) is 2

    >>> assert find_first_vowel_index("АОРТА".decode("utf8")) is 1

    >>> assert find_first_vowel_index("прнс".decode("utf8")) is None
    """

    index = word.lower().find(u"ё")

    if index is not -1:
        return index

    found = False

    for index, char in enumerate(word):
        if char.lower() in VOWELS.lower():
            if not found:
                found = True
        else:
            if found:
                break

    return index - 1 if found else None

def word_is_eligible(word, vowel_threshold=1):
    """
    >>> assert word_is_eligible("учёт".decode("utf8"), 2) is True

    >>> assert word_is_eligible("откос".decode("utf8"), 2) is True

    >>> assert word_is_eligible("молоко".decode("utf8"), 4) is False
    """

    if any(sub.lower() in word.lower() for sub in SUBSTITUTES.itervalues()):
        return False

    vowel_count = 0

    for vowel in VOWELS.lower():
        vowel_count += word.lower().count(vowel.lower())

    return vowel_count >= vowel_threshold

def generate_nonce_word(word, preserve_case=False):
    """
    >>> nw = generate_nonce_word("ЛАЛКА".decode("utf8"), preserve_case=True)
    >>> assert nw == "ХУЯЛКА".decode("utf8")

    >>> nw = generate_nonce_word("Лалка!".decode("utf8"), preserve_case=True)
    >>> assert nw == "Хуялка!".decode("utf8")

    >>> nw = generate_nonce_word("ЛАЛКА".decode("utf8"), preserve_case=False)
    >>> assert nw == "хуялка".decode("utf8")

    >>> nw = generate_nonce_word("нармкрч".decode("utf-8"), preserve_case=False)
    >>> assert nw == "хуярмкрч".decode("utf-8")
    """

    index = find_first_vowel_index(word)

    if index is None:
        return word

    char = word[index]
    nonce_word = u"{0}{1}".format(SUBSTITUTES[char.lower()], word[index + 1:])

    if not preserve_case:
        return nonce_word.lower()

    if word[:index + 1].isupper():
        nonce_word = nonce_word.upper()
    elif word[:index + 1].istitle():
        nonce_word = nonce_word.title()

    return nonce_word

def generate_nonce_phrase(phrase,
                          nonce_quantity=1.0,
                          extra_words=EXTRA_WORDS,
                          templates_simple=TEMPLATES_SIMPLE,
                          templates_composite=TEMPLATES_COMPOSITE,
                          templates_nonce_only=TEMPLATES_NONCE_ONLY,
                          preserve_case=True):
    """
    >>> templates_simple = ["{word}-{nonce_word}".decode("utf8")]
    >>> templates_composite = ["{word}-{nonce_word}".decode("utf8")]
    >>> templates_nonce_only = ["test".decode("utf8")]
    >>> extra_words = ["двач".decode("utf8")]

    >>> np = generate_nonce_phrase("ПРОВЕРКА, двач!".decode("utf8"),
    ...                            extra_words=extra_words,
    ...                            templates_simple=templates_simple,
    ...                            nonce_quantity=1.0)
    >>> assert np == "ПРОВЕРКА-ХУЁВЕРКА, двач-хуяч!".decode("utf8")

    >>> np = generate_nonce_phrase("Проверка".decode("utf8"),
    ...                            templates_composite=templates_composite,
    ...                            nonce_quantity=1.0)
    >>> assert np == "Проверка-Хуёверка".decode("utf8")

    >>> np = generate_nonce_phrase("рики-тики-тави".decode("utf8"),
    ...                            templates_nonce_only=templates_nonce_only,
    ...                            nonce_quantity=1.0)
    >>> assert np == "test-test-test".decode("utf8")

    >>> np = generate_nonce_phrase("без изменений".decode("utf8"),
    ...                            nonce_quantity=0.0)
    >>> assert np == "без изменений".decode("utf8")

    >>> np = generate_nonce_phrase("лахкрч".decode("utf-8"),
    ...                            templates_composite=templates_composite,
    ...                            nonce_quantity=1.0)
    >>> assert np == "лахкрч-хуяхкрч".decode("utf-8")
    """

    assert 0 <= nonce_quantity <= 1.0

    substitutes = OrderedDict()
    words = phrase.split()

    if len(words) is 1:
        templates = templates_composite
    else:
        templates = templates_simple

    for word in words:
        w = word.strip(u"{0}{1}".format(u"—", punctuation))
        word_parts = _translate(w, punctuation).split()

        if len(word_parts) > 1:
            templates = templates_nonce_only

        for part in word_parts:
            if not word_is_eligible(part) and part.lower() not in extra_words:
                print word_is_eligible(part)
                continue
            nonce_word = generate_nonce_word(part, preserve_case=preserve_case)
            template = choice(templates)
            sub = {part: template.format(word=part, nonce_word=nonce_word)}
            substitutes.update(sub)

    for _ in range(int(ceil(len(substitutes) * nonce_quantity))):
        match = choice(substitutes.keys())
        replace = substitutes.pop(match)
        phrase = phrase.replace(match, replace)

    return phrase

def count_vowels(word):
    """
    >>> assert count_vowels("проверка".decode("utf8")) is 3
    """

    return len([char for char in word if char.lower() in VOWELS])

class NonceGenerator(Plugin):
    """
    Herp derp.
    """

    # Quota total messages limit (int).
    QUOTA_MESSAGE_LIMIT = 3

    # Quota message interval limit in seconds (float).
    TIME_INTERVAL_LIMIT = 2

    # A value which algorithm should trigger on randomly (float 0 <= x <= 1.0)
    # Default is 1/1600.
    TRIGGER_THRESHOLD = 0.000625

    # A value which "special words only" algorithm should trigger on
    # randomly. (float 0 <= x <= 1.0). Default is 1/1.
    EXTRA_TRIGGER_THRESHOLD = 1.0

    HISTORY_LIMIT = 5

    _quotas = None

    # Previous output history for excessive flood prevention.
    _history = None

    def on_message_status(self, message, status):
        if status != cmsReceived or message.Type == cmeEmoted:
            return

        assert 0 <= self.TRIGGER_THRESHOLD <= 1.0

        assert 0 <= self.EXTRA_TRIGGER_THRESHOLD <= 1.0

        _output = list()

        if self._history is None:
            self._history = dict()

        if message.Chat.Name not in self._history:
            self._history[message.Chat.Name] = deque(maxlen=self.HISTORY_LIMIT)

        if self._quotas is None:
            self._quotas = dict()

        if message.FromHandle not in self._quotas:
            self._quotas[message.FromHandle] = list()

        # Algorithm which only triggers on certain keywords.
        if any(word.lower() in message.Body.lower() for word in EXTRA_WORDS):
            words = message.Body.split()
            for word in words:
                word = word.strip(u"{0}{1}".format(u"—", punctuation))
                for keyword in EXTRA_WORDS:
                    if word.lower().startswith(keyword.lower()):
                        if self.EXTRA_TRIGGER_THRESHOLD - uniform(0.0, 1.0) > 0:
                            _output.append(generate_nonce_phrase(word))

        # Quota algorithm.
        # Triggers on several subsequent messages are being sent by the same
        # user within the quota limits.
        quota_is_reached = False

        timestamps = self._quotas[message.FromHandle]

        if not timestamps:
            timestamps.append(message.Timestamp)
        else:
            if message.Timestamp - timestamps[-1] <= self.TIME_INTERVAL_LIMIT:
                timestamps.append(message.Timestamp)
                if len(timestamps) > self.QUOTA_MESSAGE_LIMIT:
                    quota_is_reached = True
            else:
                self._quotas[message.FromHandle] = [message.Timestamp]

        if quota_is_reached:
            result = generate_nonce_phrase(message.Body)
            if message.Body.lower() != result.lower():
                _output.append(result)

        # Algorithm which triggers randomly if nothing else has been
        # triggered before.
        if not _output:
            if self.TRIGGER_THRESHOLD - uniform(0.0, 1.0) > 0:
                result = generate_nonce_phrase(phrase=message.Body,
                                               nonce_quantity=uniform(0.1, 0.9))
                if message.Body.lower() != result.lower():
                    _output.append(result)

        msg = list()

        if _output:
            history = self._history[message.Chat.Name]
            for string_ in _output:
                if string_ not in history:
                    msg.append(string_)
                    history.append(string_)

        if msg:
            shuffle(msg)
            message.Chat.SendMessage(u"\n".join(msg))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Bug fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`noncegenerator` --- Nonce bullshit generator
==================================================
"""

__docformat__ = "restructuredtext en"

from string import punctuation
from collections import OrderedDict, deque
from random import choice, shuffle, uniform
from math import ceil

from Skype4Py.enums import cmsReceived, cmeEmoted

from plugin import Plugin

VOWELS = u"аеёиоуыэюя"

SUBSTITUTES = {
    u"а": u"хуя",
    u"е": u"хуе",
    u"ё": u"хуё",
    u"и": u"хуи",
    u"о": u"хуё",
    u"у": u"хую",
    u"ы": u"хуи",
    u"э": u"хуэ",
    u"ю": u"хую",
    u"я": u"хуя",
}

EXTRA_WORDS = [
    # u"тян",
    # u"кун",
    u"лал",
    u"лойс",
    u"азаза",
    u"нарм",
    u"бахает",
    u"бахнет",
    u"бахнул",
    u"лах",
    u"бомбанул",
    u"бамбан",
    u"пукан",
    u"пердак",
    u"выйгр",
    u"сасай",
    u"сасал",
    u"саси",
    u"сасн",
    u"здело",
]

TEMPLATES_SIMPLE = [
    u"{nonce_word}",
    u"{word}-{nonce_word}",
]

TEMPLATES_COMPOSITE = [
    u"{word}-{nonce_word}",
    u"{nonce_word}",
    u"{word}, блядь, {nonce_word}",
    u"{nonce_word}, блядь",
    u"{nonce_word}, нахуй",
    u"{word}-{nonce_word}, блядь",
    u"{word}-{nonce_word}, нахуй",
]

TEMPLATES_NONCE_ONLY = [
    u"{nonce_word}",
]

def _translate(s, characters, translate_to=u" "):
    translation_table = dict((ord(char), translate_to) for char in characters)
    return s.translate(translation_table)

def find_first_vowel_index(word):
    """
    >>> assert find_first_vowel_index("учёт".decode("utf8")) is 2

    >>> assert find_first_vowel_index("АОРТА".decode("utf8")) is 1

    >>> assert find_first_vowel_index("прнс".decode("utf8")) is None
    """

    index = word.lower().find(u"ё")

    if index is not -1:
        return index

    found = False

    for index, char in enumerate(word):
        if char.lower() in VOWELS.lower():
            if not found:
                found = True
        else:
            if found:
                break

    return index - 1 if found else None

def word_is_eligible(word, vowel_threshold=1):
    """
    >>> assert word_is_eligible("учёт".decode("utf8"), 2) is True

    >>> assert word_is_eligible("откос".decode("utf8"), 2) is True

    >>> assert word_is_eligible("молоко".decode("utf8"), 4) is False
    """

    if any(sub.lower() in word.lower() for sub in SUBSTITUTES.itervalues()):
        return False

    vowel_count = 0

    for vowel in VOWELS.lower():
        vowel_count += word.lower().count(vowel.lower())

    return vowel_count >= vowel_threshold

def generate_nonce_word(word, preserve_case=False):
    """
    >>> nw = generate_nonce_word("ЛАЛКА".decode("utf8"), preserve_case=True)
    >>> assert nw == "ХУЯЛКА".decode("utf8")

    >>> nw = generate_nonce_word("Лалка!".decode("utf8"), preserve_case=True)
    >>> assert nw == "Хуялка!".decode("utf8")

    >>> nw = generate_nonce_word("ЛАЛКА".decode("utf8"), preserve_case=False)
    >>> assert nw == "хуялка".decode("utf8")

    >>> nw = generate_nonce_word("нармкрч".decode("utf-8"), preserve_case=False)
    >>> assert nw == "хуярмкрч".decode("utf-8")
    """

    index = find_first_vowel_index(word)

    if index is None:
        return word

    char = word[index]
    nonce_word = u"{0}{1}".format(SUBSTITUTES[char.lower()], word[index + 1:])

    if not preserve_case:
        return nonce_word.lower()

    if word[:index + 1].isupper():
        nonce_word = nonce_word.upper()
    elif word[:index + 1].istitle():
        nonce_word = nonce_word.title()

    return nonce_word

def generate_nonce_phrase(phrase,
                          nonce_quantity=1.0,
                          extra_words=EXTRA_WORDS,
                          templates_simple=TEMPLATES_SIMPLE,
                          templates_composite=TEMPLATES_COMPOSITE,
                          templates_nonce_only=TEMPLATES_NONCE_ONLY,
                          preserve_case=True):
    """
    >>> templates_simple = ["{word}-{nonce_word}".decode("utf8")]
    >>> templates_composite = ["{word}-{nonce_word}".decode("utf8")]
    >>> templates_nonce_only = ["test".decode("utf8")]
    >>> extra_words = ["двач".decode("utf8")]

    >>> np = generate_nonce_phrase("ПРОВЕРКА, двач!".decode("utf8"),
    ...                            extra_words=extra_words,
    ...                            templates_simple=templates_simple,
    ...                            nonce_quantity=1.0)
    >>> assert np == "ПРОВЕРКА-ХУЁВЕРКА, двач-хуяч!".decode("utf8")

    >>> np = generate_nonce_phrase("Проверка".decode("utf8"),
    ...                            templates_composite=templates_composite,
    ...                            nonce_quantity=1.0)
    >>> assert np == "Проверка-Хуёверка".decode("utf8")

    >>> np = generate_nonce_phrase("рики-тики-тави".decode("utf8"),
    ...                            templates_nonce_only=templates_nonce_only,
    ...                            nonce_quantity=1.0)
    >>> assert np == "test-test-test".decode("utf8")

    >>> np = generate_nonce_phrase("без изменений".decode("utf8"),
    ...                            nonce_quantity=0.0)
    >>> assert np == "без изменений".decode("utf8")

    >>> np = generate_nonce_phrase("а лахкрч".decode("utf-8"),
    ...                            templates_simple=templates_simple,
    ...                            nonce_quantity=1.0)
    >>> assert np == "а лахкрч-хуяхкрч".decode("utf-8")
    """

    assert 0 <= nonce_quantity <= 1.0

    substitutes = OrderedDict()
    words = phrase.split()

    if len(words) is 1:
        templates = templates_composite
    else:
        templates = templates_simple

    for word in words:
        w = word.strip(u"{0}{1}".format(u"—", punctuation))
        word_parts = _translate(w, punctuation).split()

        if len(word_parts) > 1:
            templates = templates_nonce_only

        for part in word_parts:
            if len(part) is 1:
                continue
            if not word_is_eligible(part) and part.lower() not in extra_words:
                continue
            nonce_word = generate_nonce_word(part, preserve_case=preserve_case)
            template = choice(templates)
            sub = {part: template.format(word=part, nonce_word=nonce_word)}
            substitutes.update(sub)

    for _ in range(int(ceil(len(substitutes) * nonce_quantity))):
        match = choice(substitutes.keys())
        replace = substitutes.pop(match)
        phrase = phrase.replace(match, replace)

    return phrase

def count_vowels(word):
    """
    >>> assert count_vowels("проверка".decode("utf8")) is 3
    """

    return len([char for char in word if char.lower() in VOWELS])

class NonceGenerator(Plugin):
    """
    Herp derp.
    """

    # Quota total messages limit (int).
    QUOTA_MESSAGE_LIMIT = 3

    # Quota message interval limit in seconds (float).
    TIME_INTERVAL_LIMIT = 2

    # A value which algorithm should trigger on randomly (float 0 <= x <= 1.0)
    # Default is 1/1600.
    TRIGGER_THRESHOLD = 0.000625

    # A value which "special words only" algorithm should trigger on
    # randomly. (float 0 <= x <= 1.0). Default is 1/1.
    EXTRA_TRIGGER_THRESHOLD = 1.0

    HISTORY_LIMIT = 5

    _quotas = None

    # Previous output history for excessive flood prevention.
    _history = None

    def on_message_status(self, message, status):
        if status != cmsReceived or message.Type == cmeEmoted:
            return

        assert 0 <= self.TRIGGER_THRESHOLD <= 1.0

        assert 0 <= self.EXTRA_TRIGGER_THRESHOLD <= 1.0

        _output = list()

        if self._history is None:
            self._history = dict()

        if message.Chat.Name not in self._history:
            self._history[message.Chat.Name] = deque(maxlen=self.HISTORY_LIMIT)

        if self._quotas is None:
            self._quotas = dict()

        if message.FromHandle not in self._quotas:
            self._quotas[message.FromHandle] = list()

        # Algorithm which only triggers on certain keywords.
        if any(word.lower() in message.Body.lower() for word in EXTRA_WORDS):
            words = message.Body.split()
            for word in words:
                word = word.strip(u"{0}{1}".format(u"—", punctuation))
                for keyword in EXTRA_WORDS:
                    if word.lower().startswith(keyword.lower()):
                        if self.EXTRA_TRIGGER_THRESHOLD - uniform(0.0, 1.0) > 0:
                            _output.append(generate_nonce_phrase(word))

        # Quota algorithm.
        # Triggers on several subsequent messages are being sent by the same
        # user within the quota limits.
        quota_is_reached = False

        timestamps = self._quotas[message.FromHandle]

        if not timestamps:
            timestamps.append(message.Timestamp)
        else:
            if message.Timestamp - timestamps[-1] <= self.TIME_INTERVAL_LIMIT:
                timestamps.append(message.Timestamp)
                if len(timestamps) > self.QUOTA_MESSAGE_LIMIT:
                    quota_is_reached = True
            else:
                self._quotas[message.FromHandle] = [message.Timestamp]

        if quota_is_reached:
            result = generate_nonce_phrase(message.Body)
            if message.Body.lower() != result.lower():
                _output.append(result)

        # Algorithm which triggers randomly if nothing else has been
        # triggered before.
        if not _output:
            if self.TRIGGER_THRESHOLD - uniform(0.0, 1.0) > 0:
                result = generate_nonce_phrase(phrase=message.Body,
                                               nonce_quantity=uniform(0.1, 0.9))
                if message.Body.lower() != result.lower():
                    _output.append(result)

        msg = list()

        if _output:
            history = self._history[message.Chat.Name]
            for string_ in _output:
                if string_ not in history:
                    msg.append(string_)
                    history.append(string_)

        if msg:
            shuffle(msg)
            message.Chat.SendMessage(u"\n".join(msg))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import socket
import enums

def Plot(data):
	print('butts')

def Figure(data):
	print('in figure')
	plt.figure()
	plt.show(block=False)

def serve():

	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.bind(('127.0.0.1', 54000))
	sock.listen(0)
	(client, clientAddr) = sock.accept()
	print('got connection')
	client.send(b'\x00\xFF\xFF\xFF\xFF')
	print('sent connection bytes')
	currentPayload = 10;
	currentFunction = '';
	while 1:
		# get data from client
		data = client.recv(currentPayload)

		print('got data')

		# acknowledge client
		print('len(data) = '+str(len(data)))
		client.send(b'\x00'+int.to_bytes(4, len(data), 'little', signed=False))

		currentCommand = data[0]
		print('currentCommand = '+str(currentCommand))
		if currentCommand == enums.Command.Function: # function command
			currentFunction = data[1]
			currentPayload = int.from_bytes(data[2:-1], 'little', signed=False)
			print('currentFunction = '+str(currentFunction))
			print('currentPayload = '+str(currentPayload))

		elif currentCommand == enums.Command.Data: # data command
			if currentFunction == enums.Function.Plot: # plot function
				Plot(data)
			elif currentFunction == enums.Function.Figure: # figure function
				Figure(data)

		#print(data)

if __name__ == '__main__':
	''' plt.figure()
	plt.show(block=False)
	plt.plot([1,2,3,4])
	plt.show(block=False)
	plt.ylabel('some numbers')
	plt.show() '''
	serve()
<commit_msg>did some stuff...yeah<commit_after>#!/usr/bin/env python3

import matplotlib.pyplot as plt
import socket
import enums

def Plot(data):
	print('butts')

def Figure(data):
	print('in figure')
	plt.figure()
	plt.show(block=False)

def serve():

	print('Waiting for connection')

	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.bind(('127.0.0.1', 54000))
	sock.listen(0)
	(client, clientAddr) = sock.accept()
	print('got connection')
	client.send(b'\x00\xFF\xFF\xFF\xFF')
	print('sent connection bytes')
	currentPayload = 10;
	currentFunction = ''

	while 1:
		# get data from client
		data = client.recv(currentPayload)

		print('got data')

		# acknowledge client
		print('len(data) = '+str(len(data)))
		client.send(b'\x00'+int.to_bytes(4, len(data), 'little', signed=False))

		currentCommand = data[0]
		print('currentCommand = '+str(currentCommand))
		if currentCommand == enums.Command.Function: # function command
			currentFunction = data[1]
			currentPayload = int.from_bytes(data[2:-1], 'little', signed=False)
			print('currentFunction = '+str(currentFunction))
			print('currentPayload = '+str(currentPayload))

		elif currentCommand == enums.Command.Data: # data command
			if currentFunction == enums.Function.Plot: # plot function
				Plot(data)
			elif currentFunction == enums.Function.Figure: # figure function
				Figure(data)

		#print(data)

if __name__ == '__main__':
	''' plt.figure()
	plt.show(block=False)
	plt.plot([1,2,3,4])
	plt.show(block=False)
	plt.ylabel('some numbers')
	plt.show() '''
	serve()
<|endoftext|>


<commit_before>import os
from setuptools import setup

version = "0.0.10"

here = os.path.abspath(os.path.dirname(__file__))

install_requires = [
    "gearbox>=0.0.6"
]

try:
    README = open(os.path.join(here, 'README.rst')).read()
except IOError:
    README = ''

setup(name='axemas',
      version=version,
      url='axemas.readthedocs.org',
      keywords="axemas quickstart",
      author='AXANT',
      author_email='info@axant.it',
      license='MIT',
      install_requires=install_requires,
      packages=['axemas',],
      long_description=README,
      description='Quickstart script for gearbox that setups a new mobile project using AXEMAS.',
      zip_safe=False,
      classifiers=[
          "Framework :: Setuptools Plugin"
      ],
      entry_points={
          'gearbox.commands': [
              'axemas-quickstart = axemas.command:MakeBaseProject',
              'axemas-serve = axemas.server:ServeProjectCommand'
          ]
      },
      include_package_data=True,
      package_data={'axemas': ['commands/axemas/template/*/*/*/*/*/*/*/*']}
     )
<commit_msg>Version 0.0.13<commit_after>import os
from setuptools import setup

version = "0.0.13"

here = os.path.abspath(os.path.dirname(__file__))

install_requires = [
    "gearbox>=0.0.6"
]

try:
    README = open(os.path.join(here, 'README.rst')).read()
except IOError:
    README = ''

setup(name='axemas',
      version=version,
      url='axemas.readthedocs.org',
      keywords="axemas quickstart",
      author='AXANT',
      author_email='info@axant.it',
      license='MIT',
      install_requires=install_requires,
      packages=['axemas',],
      long_description=README,
      description='Quickstart script for gearbox that setups a new mobile project using AXEMAS.',
      zip_safe=False,
      classifiers=[
          "Framework :: Setuptools Plugin"
      ],
      entry_points={
          'gearbox.commands': [
              'axemas-quickstart = axemas.command:MakeBaseProject',
              'axemas-serve = axemas.server:ServeProjectCommand'
          ]
      },
      include_package_data=True,
      package_data={'axemas': ['commands/axemas/template/*/*/*/*/*/*/*/*']}
     )
<|endoftext|>


<commit_before>from lrrbot import bot
import storage
import utils
from commands.game import completed, game_name
from commands.show import show_name

re_stats = "|".join(storage.data["stats"])

def stat_update(lrrbot, stat, n, set_=False):
	game = lrrbot.get_current_game(readonly=False)
	if game is None:
		return None
	game.setdefault("stats", {}).setdefault(stat, 0)
	if set_:
		game["stats"][stat] = n
	else:
		game["stats"][stat] += n
	storage.save()
	return game

@bot.command("(%s)" % re_stats)
@utils.throttle(30, notify=True, params=[4])
def increment(lrrbot, conn, event, respond_to, stat):
	stat = stat.lower()
	if stat == "completed":
		completed(lrrbot, conn, event, respond_to)
		return
	game = stat_update(lrrbot, stat, 1)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game, with_emote=True)

@bot.command("(%s) add( \d+)?" % re_stats)
@utils.mod_only
def add(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, n)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s) remove( \d+)?" % re_stats)
@utils.mod_only
def remove(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, -n)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s) set (\d+)" % re_stats)
@utils.mod_only
def stat_set(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, n, True)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s)count" % re_stats)
@utils.throttle(params=[4])
def stat_print(lrrbot, conn, event, respond_to, stat, game=None, with_emote=False):
	stat = stat.lower()
	if game is None:
		game = lrrbot.get_current_game()
		if game is None:
			conn.privmsg(respond_to, "Not currently playing any game")
			return
	count = game.get("stats", {}).get(stat, 0)
	show = lrrbot.show_override or lrrbot.show
	games = storage.data.get("shows", {}).get(show, {}).get("games", {})
	countT = sum(game.get("stats", {}).get(stat, 0) for game in games.values())
	stat_details = storage.data["stats"][stat]
	display = stat_details.get("singular", stat) if count == 1 else stat_details.get("plural", stat + "s")
	if with_emote and stat_details.get("emote"):
		emote = stat_details["emote"] + " "
	else:
		emote = ""
	conn.privmsg(respond_to, "%s%d %s for %s on %s" % (emote, count, display, game_name(game), show_name(show)))
	if countT == 1000:
		conn.privmsg(respond_to, "Watch and pray for another %d %s!" % (countT, display))
	if countT == 2500:
		conn.privmsg(respond_to, "For hitting this milestone here's some code drop, Fez: 63BT5-GQ72Z-6L068 Mark of the Ninja: C03YW-YX0CA-IYC85 FTL: 5539N-PAREC-J4YGM Bastion: TE4M2-A5V64-BITNV" % (countT, display))

@bot.command("total(%s)s?" % re_stats)
@utils.throttle(params=[4])
def printtotal(lrrbot, conn, event, respond_to, stat):
	stat = stat.lower()
	count = 0
	for show in storage.data.get("shows", {}).values():
	    count += sum(game.get("stats", {}).get(stat, 0) for game in show.get("games", {}).values())
	display = storage.data["stats"][stat]
	display = display.get("singular", stat) if count == 1 else display.get("plural", stat + "s")
	conn.privmsg(respond_to, "%d total %s" % (count, display))
<commit_msg>Don't really need these hanging around<commit_after>from lrrbot import bot
import storage
import utils
from commands.game import completed, game_name
from commands.show import show_name

re_stats = "|".join(storage.data["stats"])

def stat_update(lrrbot, stat, n, set_=False):
	game = lrrbot.get_current_game(readonly=False)
	if game is None:
		return None
	game.setdefault("stats", {}).setdefault(stat, 0)
	if set_:
		game["stats"][stat] = n
	else:
		game["stats"][stat] += n
	storage.save()
	return game

@bot.command("(%s)" % re_stats)
@utils.throttle(30, notify=True, params=[4])
def increment(lrrbot, conn, event, respond_to, stat):
	stat = stat.lower()
	if stat == "completed":
		completed(lrrbot, conn, event, respond_to)
		return
	game = stat_update(lrrbot, stat, 1)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game, with_emote=True)

@bot.command("(%s) add( \d+)?" % re_stats)
@utils.mod_only
def add(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, n)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s) remove( \d+)?" % re_stats)
@utils.mod_only
def remove(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, -n)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s) set (\d+)" % re_stats)
@utils.mod_only
def stat_set(lrrbot, conn, event, respond_to, stat, n):
	stat = stat.lower()
	n = 1 if n is None else int(n)
	game = stat_update(lrrbot, stat, n, True)
	if game is None:
		conn.privmsg(respond_to, "Not currently playing any game")
		return
	stat_print.__wrapped__(lrrbot, conn, event, respond_to, stat, game)

@bot.command("(%s)count" % re_stats)
@utils.throttle(params=[4])
def stat_print(lrrbot, conn, event, respond_to, stat, game=None, with_emote=False):
	stat = stat.lower()
	if game is None:
		game = lrrbot.get_current_game()
		if game is None:
			conn.privmsg(respond_to, "Not currently playing any game")
			return
	count = game.get("stats", {}).get(stat, 0)
	show = lrrbot.show_override or lrrbot.show
	stat_details = storage.data["stats"][stat]
	display = stat_details.get("singular", stat) if count == 1 else stat_details.get("plural", stat + "s")
	if with_emote and stat_details.get("emote"):
		emote = stat_details["emote"] + " "
	else:
		emote = ""
	conn.privmsg(respond_to, "%s%d %s for %s on %s" % (emote, count, display, game_name(game), show_name(show)))

@bot.command("total(%s)s?" % re_stats)
@utils.throttle(params=[4])
def printtotal(lrrbot, conn, event, respond_to, stat):
	stat = stat.lower()
	count = 0
	for show in storage.data.get("shows", {}).values():
	    count += sum(game.get("stats", {}).get(stat, 0) for game in show.get("games", {}).values())
	display = storage.data["stats"][stat]
	display = display.get("singular", stat) if count == 1 else display.get("plural", stat + "s")
	conn.privmsg(respond_to, "%d total %s" % (count, display))
<|endoftext|>


<commit_before>class APIConnectionError(Exception):
	pass

class APIError(Exception):
	def __init__(self, json, body, status, message=None):
		self.json = json
		self.body = body
		self.status = status

		if message == None and json != None:
			message = "%s: %s" % (json.get('error', ""), json.get('error_description', ''))
		else:
			message = body

		super(APIError, self).__init__(message)

class AuthenticationError(APIError):
	pass
class UnauthorizedError(APIError):
	pass
class InvalidRequestError(APIError):
	pass
class ValidationError(APIError):
	def __init__(self, json, body, status):
		msg = "%s: %s" % (json['error'], json['details'])
		super(ValidationError, self).__init__(json, body, status, msg)
<commit_msg>Make less assumptions about the error JSON format<commit_after>class APIConnectionError(Exception):
	pass

class APIError(Exception):
	def __init__(self, json, body, status, message=None):
		self.json = json
		self.body = body
		self.status = status

		if message == None:
			message = body

		super(APIError, self).__init__(message)

class AuthenticationError(APIError):
	pass
class UnauthorizedError(APIError):
	pass
class InvalidRequestError(APIError):
	pass
class ValidationError(APIError):
	def __init__(self, json, body, status):
		msg = "%s: %s" % (json['error'], json['details'])
		super(ValidationError, self).__init__(json, body, status, msg)
<|endoftext|>


<commit_before>import constants as C
import numpy as N

datatypes = {C.DT_NONE:None, # fail if unknown
             C.DT_UNKNOWN:None,
             C.DT_BINARY:N.bool8,
             C.DT_UNSIGNED_CHAR:N.uint8,
             C.DT_SIGNED_SHORT:N.int16,
             C.DT_SIGNED_INT:N.int32,
             C.DT_FLOAT:N.float32,
             C.DT_COMPLEX:None,
             C.DT_DOUBLE:N.float64,
             C.DT_RGB:None,
             C.DT_ALL:None,
             C.DT_UINT8:N.uint8,
             C.DT_INT16:N.int16,
             C.DT_INT32:N.int32,
             C.DT_FLOAT32:N.float32,
             C.DT_COMPLEX64:N.complex64,
             C.DT_FLOAT64:N.float64,
             C.DT_RGB24:None,
             C.DT_INT8:N.int8,
             C.DT_UINT16:N.uint16,
             C.DT_UINT32:N.uint32,
             C.DT_INT64:N.int64,
             C.DT_UINT64:N.uint64,
             C.DT_FLOAT128:None,
             C.DT_COMPLEX128:None,
             C.DT_COMPLEX256:None}
<commit_msg>datatypes also no longer necessary<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#    Copyright 2013, 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{'name': 'Framework Agreement Negociation',
 'version': '1.0',
 'author': 'Camptocamp',
 'maintainer': 'Camptocamp',
 'category': 'NGO',
 'complexity': 'normal',
 'depends': ['purchase_requisition',
             'purchase_requisition_bid_selection',
             'framework_agreement'],
 'description': """
Negociate framework agreement using tender process
==================================================

This will allows you to use "The calls for Bids" model
 to negociate agreement.

To to so you have too check the box "Negociate Agreement".

The module add a state "Agreement selected" on tender and PO.

These will be the final state once you have choosen
the agreement that fit your needs the best.

Once the selection is done juste use the button "Agreement selected" on tender
That will close flow of tender and related PO accordingly.

""",
 'website': 'http://www.camptocamp.com',
 'data': ['requisition_workflow.xml',
          'purchase_workflow.xml',
          'view/purchase_requisition_view.xml'],
 'demo': [],
 'test': ['test/agreement_requisition.yml'],
 'installable': True,
 'auto_install': False,
 'license': 'AGPL-3',
 'application': False,
 }
<commit_msg>improve doc<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#    Copyright 2013, 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{'name': 'Framework Agreement Negociation in the Tender',
 'version': '1.0',
 'author': 'Camptocamp',
 'maintainer': 'Camptocamp',
 'category': 'NGO',
 'complexity': 'normal',
 'depends': ['purchase_requisition',
             'purchase_requisition_bid_selection',
             'framework_agreement'],
 'description': """
Negociate framework agreement in the Tender
===========================================

This module allows you to negociate a framework agreement in the Tender.

To to so you have to check the box "Negociate Agreement".

The module add a state "Agreement selected" on the tender and on the Purchase
Order.

These will be the final state once you have chosen the agreement that best fits
your needs.

Once the selection is done just use the button "Agreement selected" on the
tender. That will close the flow of tender and the related PO accordingly.

""",
 'website': 'http://www.camptocamp.com',
 'data': ['requisition_workflow.xml',
          'purchase_workflow.xml',
          'view/purchase_requisition_view.xml'],
 'demo': [],
 'test': ['test/agreement_requisition.yml'],
 'installable': True,
 'auto_install': False,
 'license': 'AGPL-3',
 'application': False,
 }
<|endoftext|>


<commit_before>import time, re, feedparser, codecs

#date_fmt = '%a, %d %b %Y %H:%M:%S %Z'
date_fmt = '%Y-%m-%d %H:%M:%S'

try:
  parse_date = feedparser._parse_date
except AttributeError:
  parse_date = feedparser.parse_date

def normalize(item, f):
  # get rid of RDF lossage...
  for key in ['title', 'link', 'date', 'modified', 'creator',
              'content', 'content_encoded', 'description']:
    if type(item.get(key)) == list and len(item[key]) == 1:
      item[key] = item[key][0]
    if type(item.get(key)) == dict and 'value' in item[key]:
      item[key] = item[key]['value']
  # title
  if 'title' not in item:
    item['title'] = 'Untitled'
  if type(item['title']) not in [str, unicode]:
    print 'TITLE' * 15
    import code
    from sys import exit
    code.interact(local=locals())
  # link
  if 'link' not in item:
    print 'E' * 16, 'no link in ', item
    item['link'] = f['channel']['link']
  if type(item['link']) == unicode:
    item['link'] = str(item['link'])
  if type(item['link']) != str:
    print 'LINK' * 18
    import code
    from sys import exit
    code.interact(local=locals())
  # creator
  if 'creator' not in item or item['creator'] == 'Unknown':
    item['creator'] = 'Unknown'
    if 'creator' in f['channel']:
      item['creator'] = f['channel']['creator']
  # modified date
  if 'modified' not in item:
    item['modified'] = f['channel'].get('modified')
  # created - use modified if not available
  if 'date' not in item:
    if 'modified' in item:
      date = parse_date(item['modified'])
    else:
      date = None
  else:
    date = parse_date(item['date'])
  if not date:
    # XXX use HTTP last-modified date here
    date = time.gmtime()
  item['date'] = time.strftime(date_fmt, date)
  # finish modified date
  if item['modified']:
    modified = parse_date(item['modified'])
    # add a fudge factor within modifications are not counted as such
    # 10 minutes here
    if not modified or abs(time.mktime(modified) - time.mktime(date)) < 600:
      item['modified'] = None
    else:
      item['modified'] = time.strftime(date_fmt, modified)
  # content
  if 'content' in item:
    content = item['content']
  elif 'content_encoded' in item:
    content = item['content_encoded']
  elif 'description' in item:
    content = item['description']
  else:
    content = '' + item['title'] + ''
  # balance tags like ...
  content_lc = content.lower()
  for tag in ['', '', '', '', '<font']:
    end_tag = '</' + tag[1:]
    imbalance = content_lc.count(tag) - content_lc.count(end_tag)
    if imbalance > 0:
      if '>' not in end_tag:
        end_tag += '>'
      content += end_tag * imbalance
  item['content'] = content
  # map unicode
  for key in ['title', 'link', 'date', 'modified', 'creator', 'content']:
    if type(item.get(key)) == unicode:
      item[key] = item[key].encode('ascii', 'xmlcharrefreplace')

<commit_msg>Balance  tags as well<commit_after>import time, re, feedparser, codecs

# XXX TODO
# XXX normalize feed['title'] to quote & "

#date_fmt = '%a, %d %b %Y %H:%M:%S %Z'
date_fmt = '%Y-%m-%d %H:%M:%S'

try:
  parse_date = feedparser._parse_date
except AttributeError:
  parse_date = feedparser.parse_date

def normalize(item, f):
  # get rid of RDF lossage...
  for key in ['title', 'link', 'date', 'modified', 'creator',
              'content', 'content_encoded', 'description']:
    if type(item.get(key)) == list and len(item[key]) == 1:
      item[key] = item[key][0]
    if type(item.get(key)) == dict and 'value' in item[key]:
      item[key] = item[key]['value']
  # title
  if 'title' not in item:
    item['title'] = 'Untitled'
  if type(item['title']) not in [str, unicode]:
    print 'TITLE' * 15
    import code
    from sys import exit
    code.interact(local=locals())
  # link
  if 'link' not in item:
    print 'E' * 16, 'no link in ', item
    item['link'] = f['channel']['link']
  if type(item['link']) == unicode:
    item['link'] = str(item['link'])
  if type(item['link']) != str:
    print 'LINK' * 18
    import code
    from sys import exit
    code.interact(local=locals())
  # creator
  if 'creator' not in item or item['creator'] == 'Unknown':
    item['creator'] = 'Unknown'
    if 'creator' in f['channel']:
      item['creator'] = f['channel']['creator']
  # modified date
  if 'modified' not in item:
    item['modified'] = f['channel'].get('modified')
  # created - use modified if not available
  if 'date' not in item:
    if 'modified' in item:
      date = parse_date(item['modified'])
    else:
      date = None
  else:
    date = parse_date(item['date'])
  if not date:
    # XXX use HTTP last-modified date here
    date = time.gmtime()
  item['date'] = time.strftime(date_fmt, date)
  # finish modified date
  if item['modified']:
    modified = parse_date(item['modified'])
    # add a fudge factor within modifications are not counted as such
    # 10 minutes here
    if not modified or abs(time.mktime(modified) - time.mktime(date)) < 600:
      item['modified'] = None
    else:
      item['modified'] = time.strftime(date_fmt, modified)
  # content
  if 'content' in item:
    content = item['content']
  elif 'content_encoded' in item:
    content = item['content_encoded']
  elif 'description' in item:
    content = item['description']
  else:
    content = '' + item['title'] + ''
  # balance tags like ...
  content_lc = content.lower()
  # XXX this will not work correctly for
  for tag in ['', '', '', '', '<font ', '<a ']:
    end_tag = '</' + tag[1:]
    if '>' not in end_tag:
      end_tag = end_tag .strip() + '>'
    imbalance = content_lc.count(tag) - content_lc.count(end_tag)
    if imbalance > 0:
      content += end_tag * imbalance
  item['content'] = content
  # map unicode
  for key in ['title', 'link', 'date', 'modified', 'creator', 'content']:
    if type(item.get(key)) == unicode:
      item[key] = item[key].encode('ascii', 'xmlcharrefreplace')

<|endoftext|>


<commit_before>#!/usr/bin/python
import unittest
import re
import os
import sys
import argparse
from instancetest import InstanceBasics
from eucaops import Eucaops
from eutester import xmlrunner

arg_credpath = None

class BFEBSBasics(InstanceBasics):
    def setUp(self, credpath=None):
        if credpath is None:
            credpath = arg_credpath
        super(BFEBSBasics, self).setUp(credpath)

    def RegisterImage(self, bfebs_img_url = "<image-url>", zone= None):
        '''Register a BFEBS snapshot'''
        if zone is None:
            zone = self.zone
        self.reservation = self.tester.run_instance(keypair=self.keypair.name, group=self.group.name, zone=zone)
        for instance in self.reservation.instances:
            self.assertTrue(self.create_attach_volume(instance, 2))
            instance.sys("curl " + bfebs_img_url + " > " + self.volume_device, timeout=800)
            snapshot = self.tester.create_snapshot(self.volume.id, waitOnProgress=10)
            self.assertTrue( re.search("completed", snapshot.status), "Snapshot did not complete"  )
            image_id = self.tester.register_snapshot(snapshot)
        self.image = self.tester.get_emi(image_id)

    def LaunchImage(self, zone= None):
        '''Launch a BFEBS image'''
        if zone is None:
         	zone = self.zone
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        self.assertTrue( self.tester.ping(self.reservation.instances[0].public_dns_name), 'Could not ping instance')

    def StopStart(self, zone = None):
        '''Launch a BFEBS instance, stop it then start it again'''
        if zone is None:
         	zone = self.zone
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        self.assertTrue(self.tester.stop_instances(self.reservation))
        self.assertFalse( self.tester.ping(self.reservation.instances[0].public_dns_name, poll_count=2), 'Was able to ping stopped instance')
        self.assertTrue(self.tester.start_instances(self.reservation))
        self.tester.debug("Waiting 20s for instance to boot")
        self.tester.sleep(20)
        self.assertTrue( self.tester.ping(self.reservation.instances[0].public_dns_name,  poll_count=2), 'Could not ping instance')

    def MultipleBFEBSInstances(self):
        """Run half of the available m1.small instances with a BFEBS image"""
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.MaxSmallInstances(self.tester.get_available_vms() / 2)

    def ChurnBFEBS(self):
        """Start instances and stop them before they are running, increase time to terminate on each iteration"""
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.Churn()

    def StaggeredInstances(self):
        '''Run a few instances concurrently'''
        from multiprocessing import Process
        self.failure = 0
        thread_pool = []
        queue_pool = []
        total = self.tester.get_available_vms() / 2
        for i in xrange(total):
            q = Queue()
            p = Process(target=self.run_testcase, args=(q))
            thread_pool.append(p)
            self.tester.debug("Starting Thread " + str(i))
            p.start()
            self.tester.sleep(2)
        for thread in thread_pool:
            thread.join()
        self.assertEqual(self.failure, 0, str(self.failure) + " Tests failed out of " + str(total))
        self.failure = 0

    def AddressIssue(self, zone = None):
        if zone is None:
    	 	     zone = self.zone
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        original_ip = ""
        for instance in self.reservation.instances:
            original_ip = instance.ip_address
            self.tester.debug("Terminating instance " + str(instance))
            instance.terminate()
        self.tester.sleep(1)
        self.BasicInstanceChecks()

    def run_testcase(self, queue,delay = 20, testcase="LaunchImage"):
        self.tester.sleep(delay)
        try:
            result = unittest.TextTestRunner(verbosity=2).run(BFEBSBasics(testcase))
        except Exception, e:
            queue.put(1)
            raise e
        if result.wasSuccessful():
            self.tester.debug("Passed test: " + testcase)
            queue.put(0)
        else:
            self.tester.debug("Failed test: " + testcase)
            queue.put(1)

if __name__ == "__main__":
    ## If given command line arguments, use them as test names to launch
    parser = argparse.ArgumentParser(description='Parse test suite arguments.')
    parser.add_argument('--credpath', default=".eucarc")
    parser.add_argument('--xml', action="store_true", default=False)
    parser.add_argument('--tests', nargs='+', default= ["RegisterImage","LaunchImage", "StopStart","MultipleBFEBSInstances","ChurnBFEBS"])
    args = parser.parse_args()
    arg_credpath = args.credpath
    for test in args.tests:
        if args.xml:
            try:
                os.mkdir("results")
            except OSError:
                pass
            file = open("results/test-" + test + "result.xml", "w")
            result = xmlrunner.XMLTestRunner(file).run(BFEBSBasics(test))
            file.close()
        else:
            result = unittest.TextTestRunner(verbosity=2).run(BFEBSBasics(test))
        if result.wasSuccessful():
            pass
        else:
            exit(1)

        <commit_msg>Add imgurl parameter to BFEBS test<commit_after>#!/usr/bin/python
import unittest
import re
import os
import sys
import argparse
from instancetest import InstanceBasics
from eucaops import Eucaops
from eutester import xmlrunner

arg_credpath = None
arg_imgurl = None

class BFEBSBasics(InstanceBasics):
    def setUp(self, credpath=None):
        if credpath is None:
            credpath = arg_credpath
        super(BFEBSBasics, self).setUp(credpath)

    def RegisterImage(self, bfebs_img_url = None, zone= None):
        '''Register a BFEBS snapshot'''
        if zone is None:
            zone = self.zone
        if bfebs_img_url is None:
            if arg_imgurl is not None:
                bfebs_img_url = arg_imgurl
            else:
                raise Exception("No image url provided when attempting to register a BFEBS image")
        self.reservation = self.tester.run_instance(keypair=self.keypair.name, group=self.group.name, zone=zone)
        for instance in self.reservation.instances:
            self.assertTrue(self.create_attach_volume(instance, 2))
            instance.sys("curl " + bfebs_img_url + " > " + self.volume_device, timeout=800)
            snapshot = self.tester.create_snapshot(self.volume.id, waitOnProgress=10)
            self.assertTrue( re.search("completed", snapshot.status), "Snapshot did not complete"  )
            image_id = self.tester.register_snapshot(snapshot)
        self.image = self.tester.get_emi(image_id)

    def LaunchImage(self, zone= None):
        '''Launch a BFEBS image'''
        if zone is None:
         	zone = self.zone
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        self.assertTrue( self.tester.ping(self.reservation.instances[0].public_dns_name), 'Could not ping instance')

    def StopStart(self, zone = None):
        '''Launch a BFEBS instance, stop it then start it again'''
        if zone is None:
         	zone = self.zone
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        self.assertTrue(self.tester.stop_instances(self.reservation))
        self.assertFalse( self.tester.ping(self.reservation.instances[0].public_dns_name, poll_count=2), 'Was able to ping stopped instance')
        self.assertTrue(self.tester.start_instances(self.reservation))
        self.tester.debug("Waiting 20s for instance to boot")
        self.tester.sleep(20)
        self.assertTrue( self.tester.ping(self.reservation.instances[0].public_dns_name,  poll_count=2), 'Could not ping instance')

    def MultipleBFEBSInstances(self):
        """Run half of the available m1.small instances with a BFEBS image"""
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.MaxSmallInstances(self.tester.get_available_vms() / 2)

    def ChurnBFEBS(self):
        """Start instances and stop them before they are running, increase time to terminate on each iteration"""
        self.image = self.tester.get_emi(root_device_type="ebs")
        self.Churn()

    def StaggeredInstances(self):
        '''Run a few instances concurrently'''
        from multiprocessing import Process
        self.failure = 0
        thread_pool = []
        queue_pool = []
        total = self.tester.get_available_vms() / 2
        for i in xrange(total):
            q = Queue()
            p = Process(target=self.run_testcase, args=(q))
            thread_pool.append(p)
            self.tester.debug("Starting Thread " + str(i))
            p.start()
            self.tester.sleep(2)
        for thread in thread_pool:
            thread.join()
        self.assertEqual(self.failure, 0, str(self.failure) + " Tests failed out of " + str(total))
        self.failure = 0

    def AddressIssue(self, zone = None):
        if zone is None:
    	 	     zone = self.zone
        self.reservation = self.tester.run_instance(self.image,keypair=self.keypair.name, group=self.group.name, zone=zone)
        original_ip = ""
        for instance in self.reservation.instances:
            original_ip = instance.ip_address
            self.tester.debug("Terminating instance " + str(instance))
            instance.terminate()
        self.tester.sleep(1)
        self.BasicInstanceChecks()

    def run_testcase(self, queue,delay = 20, testcase="LaunchImage"):
        self.tester.sleep(delay)
        try:
            result = unittest.TextTestRunner(verbosity=2).run(BFEBSBasics(testcase))
        except Exception, e:
            queue.put(1)
            raise e
        if result.wasSuccessful():
            self.tester.debug("Passed test: " + testcase)
            queue.put(0)
        else:
            self.tester.debug("Failed test: " + testcase)
            queue.put(1)

if __name__ == "__main__":
    ## If given command line arguments, use them as test names to launch
    parser = argparse.ArgumentParser(description='Parse test suite arguments.')
    parser.add_argument('--credpath', default=".eucarc")
    parser.add_argument('--xml', action="store_true", default=False)
    parser.add_argument('--tests', nargs='+', default= ["RegisterImage","LaunchImage", "StopStart","MultipleBFEBSInstances","ChurnBFEBS"])
    parser.add_argument('--imgurl')
    args = parser.parse_args()
    arg_credpath = args.credpath
    arg_imgurl = args.imgurl
    for test in args.tests:
        if args.xml:
            try:
                os.mkdir("results")
            except OSError:
                pass
            file = open("results/test-" + test + "result.xml", "w")
            result = xmlrunner.XMLTestRunner(file).run(BFEBSBasics(test))
            file.close()
        else:
            result = unittest.TextTestRunner(verbosity=2).run(BFEBSBasics(test))
        if result.wasSuccessful():
            pass
        else:
            exit(1)

        <|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import unicode_literals
from __future__ import division

from pyLibrary import convert, strings
from pyLibrary.debugs.logs import Log
from pyLibrary.debugs.profiles import Profiler
from pyLibrary.env import http
from pyLibrary.env.git import get_git_revision
from pyLibrary.dot import Dict, wrap, Null, coalesce, listwrap
from pyLibrary.maths import Math
from pyLibrary.times.dates import Date
from testlog_etl import etl2key
from testlog_etl.imports.hg_mozilla_org import DEFAULT_LOCALE
from testlog_etl.imports.repos.changesets import Changeset
from testlog_etl.imports.repos.revisions import Revision

DEBUG = True

def process(source_key, source, destination, resources, please_stop=None):
    lines = source.read_lines()

    etl_header = convert.json2value(lines[0])
    if etl_header.etl:
        start = 0
    elif etl_header.locale or etl_header._meta:
        start = 0
    else:
        start = 1

    keys = []
    records = []
    stats = Dict()
    for i, line in enumerate(lines[start:]):
        pulse_record = Null
        try:
            pulse_record = scrub_pulse_record(source_key, i, line, stats)
            if not pulse_record:
                continue

            with Profiler("transform_buildbot"):
                record = transform_buildbot(pulse_record.payload, resources=resources)
                record.etl = {
                    "id": i,
                    "source": pulse_record.etl,
                    "type": "join",
                    "revision": get_git_revision()
                }
            key = etl2key(record.etl)
            keys.append(key)
            records.append({"id": key, "value": record})
        except Exception, e:
            Log.warning("Problem with pulse payload {{pulse|json}}", pulse=pulse_record.payload, cause=e)
    destination.extend(records)
    return keys

def scrub_pulse_record(source_key, i, line, stats):
    """
    DUE TO MANY ETL FORMATS, THIS IS REQUIRED TO
    TURN RAW LINE INTO A STANDARD PULSE RECORD
    """
    try:
        line = strings.strip(line)
        if not line:
            return None
        pulse_record = convert.json2value(line)
        if pulse_record._meta:
            pulse_record.etl.source.id = pulse_record.etl.source.count  # REMOVE AFTER JULY 1 2015, JUST A FEW RECORDS HAVE THIS PROBLEM
            return pulse_record
        elif pulse_record.locale:
            stats.num_missing_envelope += 1
            pulse_record = wrap({
                "payload": pulse_record,
                "etl": pulse_record.etl
            })
            return pulse_record
        else:
            if i == 0 and pulse_record.source:
                #OLD-STYLE ETL HAD A HEADER RECORD
                return None

            Log.warning(
                "Line {{index}}: Do not know how to handle line for key {{key}}\n{{line}}",
                line=line,
                index=i,
                key=source_key
            )
            return None
    except Exception, e:
        Log.warning(
            "Line {{index}}: Problem with line for key {{key}}\n{{line}}",
            line=line,
            index=i,
            key=source_key,
            cause=e
        )

def transform_buildbot(payload, resources, filename=None):
    output = Dict()
    output.run.files = payload.blobber_files
    output.build.date = payload.builddate
    output.build.name = payload.buildername
    output.build.id = payload.buildid
    output.build.type = payload.buildtype
    output.build.url = payload.buildurl
    output.run.job_number = payload.job_number

    # TODO: THESE SHOULD BE ETL PROPERTIES
    output.run.insertion_time = payload.insertion_time
    output.run.key = payload.key

    output.build.locale = fix_locale(payload.locale)
    output.run.logurl = payload.logurl
    output.run.machine.os = payload.os
    output.machine.os = payload.os
    output.build.platform = payload.platform
    output.build.product = payload.product
    output.build.release = payload.release
    output.build.revision = payload.revision
    output.build.revision12 = payload.revision[0:12]
    output.run.machine.name = payload.slave
    output.machine.name = payload.slave

    # payload.status IS THE BUILDBOT STATUS
    # https://github.com/mozilla/pulsetranslator/blob/acf495738f8bd119f64820958c65e348aa67963c/pulsetranslator/pulsetranslator.py#L295
    # https://hg.mozilla.org/build/buildbot/file/fbfb8684802b/master/buildbot/status/builder.py#l25
    output.run.status = payload.status   # TODO: REMOVE EVENTUALLY
    try:
        output.run.buildbot_status = {
            0: "success",
            1: "warnings",
            2: "failure",
            3: "skipped",
            4: "exception",
            5: "retry",
            6: "cancelled",
            None: None
        }[payload.status]
    except Exception, e:
        Log.warning("It seems the Pulse payload status {{status|quote}} has no string representative", status=payload.status)

    output.run.talos = payload.talos
    output.run.suite = payload.test
    output.run.timestamp = Date(payload.timestamp).unix
    output.build.branch = payload.tree

    # JUST IN CASE THERE ARE MORE PROPERTIES
    output.other = payload = payload.copy()
    payload.blobber_files = None
    payload.builddate = None
    payload.buildername = None
    payload.buildid = None
    payload.buildtype = None
    payload.buildurl = None
    payload.etl = None
    payload.insertion_time = None
    payload.job_number = None
    payload.key = None
    payload.locale = None
    payload.logurl = None
    payload.os = None
    payload.platform = None
    payload.product = None
    payload.release = None
    payload.revision = None
    payload.slave = None
    payload.status = None
    payload.talos = None
    payload.test = None
    payload.timestamp = None
    payload.tree = None

    path = output.run.suite.split("-")
    if Math.is_integer(path[-1]):
        output.run.chunk = int(path[-1])
        output.run.suite = "-".join(path[:-1])

    output.run.files = [
        {"name": name, "url": url}
        for name, url in output.run.files.items()
        if filename is None or name == filename
    ]

    try:
        if output.build.branch:
            rev = Revision(branch={"name": output.build.branch}, changeset=Changeset(id=output.build.revision))
            output.repo = resources.hg.get_revision(rev, output.build.locale.replace("en-US", DEFAULT_LOCALE))
        elif output.other.what == "This is a heartbeat":
            pass
        else:
            Log.warning("No branch!\n{{output|indent}}", output=output)
    except Exception, e:
        Log.warning("Can not get revision for branch {{branch}}\n{{details|json|indent}}", branch=output.build.branch, details=output, cause=e)
        # resources.hg.find_changeset(output.build.revision)

    return output

def fix_locale(locale):
    # compensate for bug https://bugzilla.mozilla.org/show_bug.cgi?id=1174979
    if locale.find("\"") == -1:
        return locale
    return strings.between(locale, "\"", "\"")
<commit_msg>filter out heartbeat sooner<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import unicode_literals
from __future__ import division

from pyLibrary import convert, strings
from pyLibrary.debugs.logs import Log
from pyLibrary.debugs.profiles import Profiler
from pyLibrary.env import http
from pyLibrary.env.git import get_git_revision
from pyLibrary.dot import Dict, wrap, Null, coalesce, listwrap
from pyLibrary.maths import Math
from pyLibrary.times.dates import Date
from testlog_etl import etl2key
from testlog_etl.imports.hg_mozilla_org import DEFAULT_LOCALE
from testlog_etl.imports.repos.changesets import Changeset
from testlog_etl.imports.repos.revisions import Revision

DEBUG = True

def process(source_key, source, destination, resources, please_stop=None):
    lines = source.read_lines()

    etl_header = convert.json2value(lines[0])
    if etl_header.etl:
        start = 0
    elif etl_header.locale or etl_header._meta:
        start = 0
    else:
        start = 1

    keys = []
    records = []
    stats = Dict()
    for i, line in enumerate(lines[start:]):
        pulse_record = Null
        try:
            pulse_record = scrub_pulse_record(source_key, i, line, stats)
            if not pulse_record:
                continue

            with Profiler("transform_buildbot"):
                record = transform_buildbot(pulse_record.payload, resources=resources)
                record.etl = {
                    "id": i,
                    "source": pulse_record.etl,
                    "type": "join",
                    "revision": get_git_revision()
                }
            key = etl2key(record.etl)
            keys.append(key)
            records.append({"id": key, "value": record})
        except Exception, e:
            Log.warning("Problem with pulse payload {{pulse|json}}", pulse=pulse_record.payload, cause=e)
    destination.extend(records)
    return keys

def scrub_pulse_record(source_key, i, line, stats):
    """
    DUE TO MANY ETL FORMATS, THIS IS REQUIRED TO
    TURN RAW LINE INTO A STANDARD PULSE RECORD
    """
    try:
        line = strings.strip(line)
        if not line:
            return None
        pulse_record = convert.json2value(line)
        if pulse_record.payload.what == "This is a heartbeat":
            return None
        elif pulse_record._meta:
            pulse_record.etl.source.id = pulse_record.etl.source.count  # REMOVE AFTER JULY 1 2015, JUST A FEW RECORDS HAVE THIS PROBLEM
            return pulse_record
        elif pulse_record.locale:
            stats.num_missing_envelope += 1
            pulse_record = wrap({
                "payload": pulse_record,
                "etl": pulse_record.etl
            })
            return pulse_record
        else:
            if i == 0 and pulse_record.source:
                #OLD-STYLE ETL HAD A HEADER RECORD
                return None

            Log.warning(
                "Line {{index}}: Do not know how to handle line for key {{key}}\n{{line}}",
                line=line,
                index=i,
                key=source_key
            )
            return None
    except Exception, e:
        Log.warning(
            "Line {{index}}: Problem with line for key {{key}}\n{{line}}",
            line=line,
            index=i,
            key=source_key,
            cause=e
        )

def transform_buildbot(payload, resources, filename=None):
    output = Dict()
    output.run.files = payload.blobber_files
    output.build.date = payload.builddate
    output.build.name = payload.buildername
    output.build.id = payload.buildid
    output.build.type = payload.buildtype
    output.build.url = payload.buildurl
    output.run.job_number = payload.job_number

    # TODO: THESE SHOULD BE ETL PROPERTIES
    output.run.insertion_time = payload.insertion_time
    output.run.key = payload.key

    output.build.locale = fix_locale(payload.locale)
    output.run.logurl = payload.logurl
    output.run.machine.os = payload.os
    output.machine.os = payload.os
    output.build.platform = payload.platform
    output.build.product = payload.product
    output.build.release = payload.release
    output.build.revision = payload.revision
    output.build.revision12 = payload.revision[0:12]
    output.run.machine.name = payload.slave
    output.machine.name = payload.slave

    # payload.status IS THE BUILDBOT STATUS
    # https://github.com/mozilla/pulsetranslator/blob/acf495738f8bd119f64820958c65e348aa67963c/pulsetranslator/pulsetranslator.py#L295
    # https://hg.mozilla.org/build/buildbot/file/fbfb8684802b/master/buildbot/status/builder.py#l25
    output.run.status = payload.status   # TODO: REMOVE EVENTUALLY
    try:
        output.run.buildbot_status = {
            0: "success",
            1: "warnings",
            2: "failure",
            3: "skipped",
            4: "exception",
            5: "retry",
            6: "cancelled",
            None: None
        }[payload.status]
    except Exception, e:
        Log.warning("It seems the Pulse payload status {{status|quote}} has no string representative", status=payload.status)

    output.run.talos = payload.talos
    output.run.suite = payload.test
    output.run.timestamp = Date(payload.timestamp).unix
    output.build.branch = payload.tree

    # JUST IN CASE THERE ARE MORE PROPERTIES
    output.other = payload = payload.copy()
    payload.blobber_files = None
    payload.builddate = None
    payload.buildername = None
    payload.buildid = None
    payload.buildtype = None
    payload.buildurl = None
    payload.etl = None
    payload.insertion_time = None
    payload.job_number = None
    payload.key = None
    payload.locale = None
    payload.logurl = None
    payload.os = None
    payload.platform = None
    payload.product = None
    payload.release = None
    payload.revision = None
    payload.slave = None
    payload.status = None
    payload.talos = None
    payload.test = None
    payload.timestamp = None
    payload.tree = None

    path = output.run.suite.split("-")
    if Math.is_integer(path[-1]):
        output.run.chunk = int(path[-1])
        output.run.suite = "-".join(path[:-1])

    output.run.files = [
        {"name": name, "url": url}
        for name, url in output.run.files.items()
        if filename is None or name == filename
    ]

    try:
        if output.build.branch:
            rev = Revision(branch={"name": output.build.branch}, changeset=Changeset(id=output.build.revision))
            output.repo = resources.hg.get_revision(rev, output.build.locale.replace("en-US", DEFAULT_LOCALE))
        else:
            Log.warning("No branch!\n{{output|indent}}", output=output)
    except Exception, e:
        Log.warning("Can not get revision for branch {{branch}}\n{{details|json|indent}}", branch=output.build.branch, details=output, cause=e)
        # resources.hg.find_changeset(output.build.revision)

    return output

def fix_locale(locale):
    # compensate for bug https://bugzilla.mozilla.org/show_bug.cgi?id=1174979
    if locale.find("\"") == -1:
        return locale
    return strings.between(locale, "\"", "\"")
<|endoftext|>