<commit_before>""" Command sets for wifi bridge version 6. """

import math
from colorsys import rgb_to_hsv

from limitlessled.group.rgbw import RGBW, RGBWW, BRIDGE_LED
from limitlessled.group.white import WHITE
from limitlessled.group.commands import CommandSet, Command

class CommandSetV6(CommandSet):
    """ Base command set for wifi bridge v6. """

    SUPPORTED_VERSIONS = [6]
    PASSWORD_BYTE1 = 0x00
    PASSWORD_BYTE2 = 0x00
    MAX_COLOR = 0xFF
    MAX_BRIGHTNESS = 0x64
    MAX_TEMPERATURE = 0x64

    def __init__(self, bridge, group_number, remote_style,
                 brightness_steps=None, color_steps=None,
                 temperature_steps=None):
        """
        Initialize the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        :param remote_style: The remote style of the device to control.
        :param brightness_steps: The number of brightness steps.
        :param color_steps: The number of color steps.
        :param temperature_steps: The number of temperature steps.
        """
        brightness_steps = brightness_steps or self.MAX_BRIGHTNESS + 1
        color_steps = color_steps or self.MAX_COLOR + 1
        temperature_steps = temperature_steps or self.MAX_TEMPERATURE + 1
        super().__init__(bridge, group_number, brightness_steps,
                         color_steps=color_steps,
                         temperature_steps=temperature_steps)
        self._remote_style = remote_style

    def convert_brightness(self, brightness):
        """
        Convert the brightness from decimal percent (0.0-1.0)
        to byte representation for use in commands.
        :param brightness: The brightness from in decimal percent (0.0-1.0).
        :return: The brightness in byte representation.
        """
        return math.ceil(brightness * self.MAX_BRIGHTNESS)

    def convert_temperature(self, temperature):
        """
        Convert the temperature from decimal percent (0.0-1.0)
        to byte representation for use in commands.
        :param temperature: The temperature from in decimal percent (0.0-1.0).
        :return: The temperature in byte representation.
        """
        return math.ceil(temperature * self.MAX_TEMPERATURE)

    def convert_color(self, color):
        """
        Converts the color from RGB to byte representation for use in commands.
        :param color: The RGB color tuple.
        :return: The color in byte representation (best-effort basis).
        """
        hue = rgb_to_hsv(*color)[0]
        cn = (176 - math.floor(hue * self.MAX_COLOR)) % (self.MAX_COLOR + 1)
        cn = self.MAX_COLOR - cn - 0x37

        return cn % (self.MAX_COLOR + 1)

    def _build_command(self, cmd_1, cmd_2):
        """
        Constructs the complete command.
        :param cmd_1: Light command 1.
        :param cmd_2: Light command 2.
        :return: The complete command.
        """
        wb1 = self._bridge.wb1
        wb2 = self._bridge.wb2
        sn = self._bridge.sn

        preamble = [0x80, 0x00, 0x00, 0x00, 0x11, wb1, wb2, 0x00, sn, 0x00]
        cmd = [0x31, self.PASSWORD_BYTE1, self.PASSWORD_BYTE2,
               self._remote_style, cmd_1, cmd_2, cmd_2, cmd_2, cmd_2]
        zone_selector = [self._group_number, 0x00]
        checksum = sum(cmd + zone_selector) & 0xFF

        return Command(preamble + cmd + zone_selector + [checksum],
                       self._group_number)

class CommandSetBridgeLightV6(CommandSetV6):
    """ Command set for bridge light of wifi bridge v6. """

    SUPPORTED_LED_TYPES = [BRIDGE_LED]
    REMOTE_STYLE = 0x00

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x03, 0x03)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x03, 0x04)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x05)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x02, self.convert_brightness(brightness))

class CommandSetWhiteV6(CommandSetV6):
    """ Command set for white led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [WHITE]
    REMOTE_STYLE = 0x08

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x04, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x04, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x04, 0x05)

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x03, self.convert_brightness(brightness))

    def temperature(self, temperature):
        """
        Build command for setting the temperature of the led.
        :param temperature: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x05, temperature)

class CommandSetRgbwV6(CommandSetV6):
    """ Command set for RGBW led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [RGBW]
    REMOTE_STYLE = 0x08

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x04, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x04, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x04, 0x05)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x05, 0x64)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x03, self.convert_brightness(brightness))

class CommandSetRgbwwV6(CommandSetV6):
    """ Command set for RGBWW led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [RGBWW]
    REMOTE_STYLE = 0x07

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x03, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x03, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x06)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x05)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x02, self.convert_brightness(brightness))
<commit_msg>Switched commands for rgbw and rgbww.<commit_after>""" Command sets for wifi bridge version 6. """

import math
from colorsys import rgb_to_hsv

from limitlessled.group.rgbw import RGBW, RGBWW, BRIDGE_LED
from limitlessled.group.white import WHITE
from limitlessled.group.commands import CommandSet, Command

class CommandSetV6(CommandSet):
    """ Base command set for wifi bridge v6. """

    SUPPORTED_VERSIONS = [6]
    PASSWORD_BYTE1 = 0x00
    PASSWORD_BYTE2 = 0x00
    MAX_COLOR = 0xFF
    MAX_BRIGHTNESS = 0x64
    MAX_TEMPERATURE = 0x64

    def __init__(self, bridge, group_number, remote_style,
                 brightness_steps=None, color_steps=None,
                 temperature_steps=None):
        """
        Initialize the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        :param remote_style: The remote style of the device to control.
        :param brightness_steps: The number of brightness steps.
        :param color_steps: The number of color steps.
        :param temperature_steps: The number of temperature steps.
        """
        brightness_steps = brightness_steps or self.MAX_BRIGHTNESS + 1
        color_steps = color_steps or self.MAX_COLOR + 1
        temperature_steps = temperature_steps or self.MAX_TEMPERATURE + 1
        super().__init__(bridge, group_number, brightness_steps,
                         color_steps=color_steps,
                         temperature_steps=temperature_steps)
        self._remote_style = remote_style

    def convert_brightness(self, brightness):
        """
        Convert the brightness from decimal percent (0.0-1.0)
        to byte representation for use in commands.
        :param brightness: The brightness from in decimal percent (0.0-1.0).
        :return: The brightness in byte representation.
        """
        return math.ceil(brightness * self.MAX_BRIGHTNESS)

    def convert_temperature(self, temperature):
        """
        Convert the temperature from decimal percent (0.0-1.0)
        to byte representation for use in commands.
        :param temperature: The temperature from in decimal percent (0.0-1.0).
        :return: The temperature in byte representation.
        """
        return math.ceil(temperature * self.MAX_TEMPERATURE)

    def convert_color(self, color):
        """
        Converts the color from RGB to byte representation for use in commands.
        :param color: The RGB color tuple.
        :return: The color in byte representation (best-effort basis).
        """
        hue = rgb_to_hsv(*color)[0]
        cn = (176 - math.floor(hue * self.MAX_COLOR)) % (self.MAX_COLOR + 1)
        cn = self.MAX_COLOR - cn - 0x37

        return cn % (self.MAX_COLOR + 1)

    def _build_command(self, cmd_1, cmd_2):
        """
        Constructs the complete command.
        :param cmd_1: Light command 1.
        :param cmd_2: Light command 2.
        :return: The complete command.
        """
        wb1 = self._bridge.wb1
        wb2 = self._bridge.wb2
        sn = self._bridge.sn

        preamble = [0x80, 0x00, 0x00, 0x00, 0x11, wb1, wb2, 0x00, sn, 0x00]
        cmd = [0x31, self.PASSWORD_BYTE1, self.PASSWORD_BYTE2,
               self._remote_style, cmd_1, cmd_2, cmd_2, cmd_2, cmd_2]
        zone_selector = [self._group_number, 0x00]
        checksum = sum(cmd + zone_selector) & 0xFF

        return Command(preamble + cmd + zone_selector + [checksum],
                       self._group_number)

class CommandSetBridgeLightV6(CommandSetV6):
    """ Command set for bridge light of wifi bridge v6. """

    SUPPORTED_LED_TYPES = [BRIDGE_LED]
    REMOTE_STYLE = 0x00

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x03, 0x03)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x03, 0x04)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x05)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x02, self.convert_brightness(brightness))

class CommandSetWhiteV6(CommandSetV6):
    """ Command set for white led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [WHITE]
    REMOTE_STYLE = 0x08

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x04, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x04, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x04, 0x05)

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x03, self.convert_brightness(brightness))

    def temperature(self, temperature):
        """
        Build command for setting the temperature of the led.
        :param temperature: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x05, temperature)

class CommandSetRgbwV6(CommandSetV6):
    """ Command set for RGBW led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [RGBW]
    REMOTE_STYLE = 0x07

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x03, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x03, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x06)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x03, 0x05)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x02, self.convert_brightness(brightness))

class CommandSetRgbwwV6(CommandSetV6):
    """ Command set for RGBWW led light connected to wifi bridge v6. """

    SUPPORTED_LED_TYPES = [RGBWW]
    REMOTE_STYLE = 0x08

    def __init__(self, bridge, group_number):
        """
        Initializes the command set.
        :param bridge: The bridge the leds are connected to.
        :param group_number: The group number.
        """
        super().__init__(bridge, group_number, self.REMOTE_STYLE)

    def on(self):
        """
        Build command for turning the led on.
        :return: The command.
        """
        return self._build_command(0x04, 0x01)

    def off(self):
        """
        Build command for turning the led off.
        :return: The command.
        """
        return self._build_command(0x04, 0x02)

    def night_light(self):
        """
        Build command for turning the led into night light mode.
        :return: The command.
        """
        return self._build_command(0x04, 0x05)

    def white(self):
        """
        Build command for turning the led into white mode.
        :return: The command.
        """
        return self._build_command(0x05, 0x64)

    def color(self, color):
        """
        Build command for setting the color of the led.
        :param color: RGB color tuple.
        :return: The command.
        """
        return self._build_command(0x01, self.convert_color(color))

    def brightness(self, brightness):
        """
        Build command for setting the brightness of the led.
        :param brightness: Value to set (0.0-1.0).
        :return: The command.
        """
        return self._build_command(0x03, self.convert_brightness(brightness))
<|endoftext|>


<commit_before>import unittest
from StringIO import StringIO
import json
import urllib
from importlib import import_module as imp
from fabric.api import env, run, sudo, put, cd
from fabric.contrib.files import exists
from path import path

cfg = {}
cfg['sarge-home'] = path('/var/local/sarge')
cfg['sarge-venv'] = path('/var/local/sarge-sandbox')

def provision():
    sudo("virtualenv '%(sarge-venv)s' --no-site-packages" % cfg)
    sudo("'%(sarge-venv)s'/bin/pip install -e /sarge-src" % cfg)
    sudo("'%(sarge-venv)s'/bin/pip install flup" % cfg)

def setUpModule(self):
    env['key_filename'] = path(__file__).parent / 'vagrant_id_rsa'
    env['host_string'] = 'vagrant@192.168.13.13'
    if not exists(cfg['sarge-venv']):
        provision()

    self._nginx_symlink = '/etc/nginx/sites-enabled/testy'
    nginx_all_sites = cfg['sarge-home'] / 'nginx.plugin' / 'sarge_sites.conf'
    sudo("ln -s '%s' '%s'" % (nginx_all_sites, self._nginx_symlink))

def tearDownModule(self):
    sudo("rm %s" % self._nginx_symlink)
    from fabric.network import disconnect_all
    disconnect_all()

def sarge_cmd(cmd):
    base = ("'%(sarge-venv)s'/bin/sarge '%(sarge-home)s' " % cfg)
    return run(base + cmd)

def supervisorctl_cmd(cmd):
    base = ("'%(sarge-venv)s'/bin/supervisorctl "
            "-c '%(sarge-home)s'/supervisord.conf " % cfg)
    return run(base + cmd)

def remote_listdir(name):
    cmd = ("python -c 'import json,os; "
           "print json.dumps(os.listdir(\"%s\"))'" % name)
    return json.loads(run(cmd))

def put_json(data, remote_path, **kwargs):
    return put(StringIO(json.dumps(data)), str(remote_path), **kwargs)

def get_url(url):
    f = urllib.urlopen(url)
    try:
        return f.read()
    finally:
        f.close()

class VagrantDeploymentTest(unittest.TestCase):

    def setUp(self):
        sudo("mkdir '%(sarge-home)s'" % cfg)
        sudo("chown vagrant: '%(sarge-home)s'" % cfg)
        put_json({'plugins': ['sarge:NginxPlugin']},
                 cfg['sarge-home'] / imp('sarge.core').SARGE_CFG,
                 use_sudo=True)
        sarge_cmd("init")
        run("'%(sarge-venv)s'/bin/supervisord "
            "-c '%(sarge-home)s'/supervisord.conf" % cfg)

    def tearDown(self):
        supervisorctl_cmd("shutdown")
        sudo("rm -rf '%(sarge-home)s'" % cfg)

    def test_ping(self):
        assert run('pwd') == '/home/vagrant'

    def test_deploy_simple_wsgi_app(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'))

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'wsgi',
                     'url': '/',
                     'app_factory': 'mytinyapp:gettheapp'},
                 ]},
                 version_folder / 'sargeapp.yaml')
        app_py = ('def gettheapp(appcfg):\n'
                  '    def theapp(environ, start_response):\n'
                  '        start_response("200 OK", [])\n'
                  '        return ["hello sarge!\\n"]\n'
                  '    return theapp\n')
        put(StringIO(app_py), str(version_folder / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge!\n")

    def test_deploy_new_app_version(self):
        app_cfg = {
            'urlmap': [
                {'type': 'wsgi',
                 'url': '/',
                 'app_factory': 'mytinyapp:gettheapp'},
            ],
        }
        app_py_tmpl = ('def gettheapp(appcfg):\n'
                       '    def theapp(environ, start_response):\n'
                       '        start_response("200 OK", [])\n'
                       '        return ["hello sarge %s!\\n"]\n'
                       '    return theapp\n')

        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        # deploy version one
        version_folder_1 = path(sarge_cmd("new_version testy"))
        put_json(app_cfg, version_folder_1 / 'sargeapp.yaml')
        put(StringIO(app_py_tmpl % 'one'),
            str(version_folder_1 / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder_1)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge one!\n")

        # deploy version two
        version_folder_2 = path(sarge_cmd("new_version testy"))
        put_json(app_cfg, version_folder_2 / 'sargeapp.yaml')
        put(StringIO(app_py_tmpl % 'two'),
            str(version_folder_2 / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder_2)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge two!\n")

    def test_deploy_php(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'php', 'url': '/'},
                 ]},
                 version_folder / 'sargeapp.yaml')

        app_php = ('<?php echo "hello from" . " PHP!\\n"; ?>')
        put(StringIO(app_php), str(version_folder / 'someapp.php'))
        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/someapp.php'),
                         "hello from PHP!\n")

    def test_deploy_static_site(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'static', 'url': '/', 'path': ''},
                 ]},
                 version_folder / 'sargeapp.yaml')

        with cd(str(version_folder)):
            run("echo 'hello static!' > hello.html")
            run("mkdir sub")
            run("echo 'submarine' > sub/marine.txt")

        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/hello.html'),
                         "hello static!\n")

        self.assertEqual(get_url('http://192.168.13.13:8013/sub/marine.txt'),
                         "submarine\n")
<commit_msg>remove any stale folders before running tests<commit_after>import unittest
from StringIO import StringIO
import json
import urllib
from importlib import import_module as imp
from fabric.api import env, run, sudo, put, cd
from fabric.contrib.files import exists
from path import path

cfg = {}
cfg['sarge-home'] = path('/var/local/sarge')
cfg['sarge-venv'] = path('/var/local/sarge-sandbox')

def provision():
    sudo("virtualenv '%(sarge-venv)s' --no-site-packages" % cfg)
    sudo("'%(sarge-venv)s'/bin/pip install -e /sarge-src" % cfg)
    sudo("'%(sarge-venv)s'/bin/pip install flup" % cfg)

def setUpModule(self):
    env['key_filename'] = path(__file__).parent / 'vagrant_id_rsa'
    env['host_string'] = 'vagrant@192.168.13.13'
    if not exists(cfg['sarge-venv']):
        provision()

    sudo("rm -rf '%(sarge-home)s'" % cfg)
    self._nginx_symlink = '/etc/nginx/sites-enabled/testy'
    nginx_all_sites = cfg['sarge-home'] / 'nginx.plugin' / 'sarge_sites.conf'
    sudo("ln -s '%s' '%s'" % (nginx_all_sites, self._nginx_symlink))

def tearDownModule(self):
    sudo("rm %s" % self._nginx_symlink)
    from fabric.network import disconnect_all
    disconnect_all()

def sarge_cmd(cmd):
    base = ("'%(sarge-venv)s'/bin/sarge '%(sarge-home)s' " % cfg)
    return run(base + cmd)

def supervisorctl_cmd(cmd):
    base = ("'%(sarge-venv)s'/bin/supervisorctl "
            "-c '%(sarge-home)s'/supervisord.conf " % cfg)
    return run(base + cmd)

def remote_listdir(name):
    cmd = ("python -c 'import json,os; "
           "print json.dumps(os.listdir(\"%s\"))'" % name)
    return json.loads(run(cmd))

def put_json(data, remote_path, **kwargs):
    return put(StringIO(json.dumps(data)), str(remote_path), **kwargs)

def get_url(url):
    f = urllib.urlopen(url)
    try:
        return f.read()
    finally:
        f.close()

class VagrantDeploymentTest(unittest.TestCase):

    def setUp(self):
        sudo("mkdir '%(sarge-home)s'" % cfg)
        sudo("chown vagrant: '%(sarge-home)s'" % cfg)
        put_json({'plugins': ['sarge:NginxPlugin']},
                 cfg['sarge-home'] / imp('sarge.core').SARGE_CFG,
                 use_sudo=True)
        sarge_cmd("init")
        run("'%(sarge-venv)s'/bin/supervisord "
            "-c '%(sarge-home)s'/supervisord.conf" % cfg)

    def tearDown(self):
        supervisorctl_cmd("shutdown")
        sudo("rm -rf '%(sarge-home)s'" % cfg)

    def test_ping(self):
        assert run('pwd') == '/home/vagrant'

    def test_deploy_simple_wsgi_app(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'))

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'wsgi',
                     'url': '/',
                     'app_factory': 'mytinyapp:gettheapp'},
                 ]},
                 version_folder / 'sargeapp.yaml')
        app_py = ('def gettheapp(appcfg):\n'
                  '    def theapp(environ, start_response):\n'
                  '        start_response("200 OK", [])\n'
                  '        return ["hello sarge!\\n"]\n'
                  '    return theapp\n')
        put(StringIO(app_py), str(version_folder / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge!\n")

    def test_deploy_new_app_version(self):
        app_cfg = {
            'urlmap': [
                {'type': 'wsgi',
                 'url': '/',
                 'app_factory': 'mytinyapp:gettheapp'},
            ],
        }
        app_py_tmpl = ('def gettheapp(appcfg):\n'
                       '    def theapp(environ, start_response):\n'
                       '        start_response("200 OK", [])\n'
                       '        return ["hello sarge %s!\\n"]\n'
                       '    return theapp\n')

        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        # deploy version one
        version_folder_1 = path(sarge_cmd("new_version testy"))
        put_json(app_cfg, version_folder_1 / 'sargeapp.yaml')
        put(StringIO(app_py_tmpl % 'one'),
            str(version_folder_1 / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder_1)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge one!\n")

        # deploy version two
        version_folder_2 = path(sarge_cmd("new_version testy"))
        put_json(app_cfg, version_folder_2 / 'sargeapp.yaml')
        put(StringIO(app_py_tmpl % 'two'),
            str(version_folder_2 / 'mytinyapp.py'))
        sarge_cmd("activate_version testy '%s'" % version_folder_2)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/'),
                         "hello sarge two!\n")

    def test_deploy_php(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'php', 'url': '/'},
                 ]},
                 version_folder / 'sargeapp.yaml')

        app_php = ('<?php echo "hello from" . " PHP!\\n"; ?>')
        put(StringIO(app_php), str(version_folder / 'someapp.php'))
        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/someapp.php'),
                         "hello from PHP!\n")

    def test_deploy_static_site(self):
        put_json({'name': 'testy',
                  'nginx_options': {'listen': '8013'}},
                 (cfg['sarge-home'] /
                    imp('sarge.core').DEPLOYMENT_CFG_DIR /
                    'testy.yaml'),
                 use_sudo=True)

        version_folder = path(sarge_cmd("new_version testy"))

        put_json({'urlmap': [
                    {'type': 'static', 'url': '/', 'path': ''},
                 ]},
                 version_folder / 'sargeapp.yaml')

        with cd(str(version_folder)):
            run("echo 'hello static!' > hello.html")
            run("mkdir sub")
            run("echo 'submarine' > sub/marine.txt")

        sarge_cmd("activate_version testy '%s'" % version_folder)
        sudo("service nginx reload")

        self.assertEqual(get_url('http://192.168.13.13:8013/hello.html'),
                         "hello static!\n")

        self.assertEqual(get_url('http://192.168.13.13:8013/sub/marine.txt'),
                         "submarine\n")
<|endoftext|>


<commit_before>import random

import numpy

from sensorimotor.general_temporal_memory import GeneralTemporalMemory
from nupic.research.monitor_mixin.temporal_memory_monitor_mixin import (
  TemporalMemoryMonitorMixin)
class MonitoredGeneralTemporalMemory(TemporalMemoryMonitorMixin,
                                     GeneralTemporalMemory): pass

from nupic.encoders.random_distributed_scalar import (
  RandomDistributedScalarEncoder)

PLOT_EVERY = 25

class Road(object):

  def get(self, distance, field):
    raise NotImplementedError

class StraightRoad(Road):

  def get(self, distance, field):
    return (float(field.width) / 2, 25)

class ZigZagRoad(Road):

  def __init__(self, width=25, zigZagEvery=60):
    super(ZigZagRoad, self).__init__()

    self.width = width
    self.zigZagEvery = zigZagEvery

  def get(self, distance, field):
    zig = int(distance / self.zigZagEvery)
    zag = int((distance + self.zigZagEvery) / self.zigZagEvery)

    zigDistance = distance % self.zigZagEvery

    numpy.random.seed(zig)
    zigPosition = numpy.random.randint(0, field.width)
    numpy.random.seed(zag)
    zagPosition = numpy.random.randint(0, field.width)
    separation = zagPosition - zigPosition

    zigPercent = float(zigDistance) / self.zigZagEvery
    position = zigPosition + (separation * zigPercent)

    return (position, self.width)

  def containsVehicle(self, vehicle):
    if vehicle.position > 50:
      return True

    return False

class Field(object):

  def __init__(self, road, width=100):
    self.width = width
    self.road = road

class Sensor(object):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def sense(self, field, vehicle):
    raise NotImplementedError

class NoOpSensor(Sensor):

  def sense(self, field, vehicle):
    return None

class PositionSensor(Sensor):

  def sense(self, field, vehicle):
    return vehicle.position

class Motor(object):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def move(self, motorValue, vehicle):
    raise NotImplementedError

class AccelerationMotor(Motor):

  def __init__(self, frictionCoefficient=0.1, noise=(0.0, 0.0)):
    self.frictionCoefficient = frictionCoefficient
    self.noise = noise

  def move(self, motorValue, vehicle):
    acceleration = motorValue + random.gauss(*self.noise)
    friction = vehicle.velocity * self.frictionCoefficient
    vehicle.velocity += acceleration - friction
    vehicle.position += vehicle.velocity
    vehicle.position = vehicle.position % vehicle.field.width

class PositionMotor(Motor):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def move(self, motorValue, vehicle):
    vehicle.position += motorValue + random.gauss(*self.noise)

class Vehicle(object):

  def __init__(self, field, sensor, motor):
    self.field = field
    self.sensor = sensor
    self.motor = motor

    self.position = 0
    self.distance = 0
    self.velocity = 0

    self.sensorValue = None
    self.motorValue = None

  def move(self):
    raise NotImplementedError()

  def tick(self):
    self.sensorValue = self.sensor.sense(self.field, self)
    self.motorValue = self.move()
    self.motor.move(self.motorValue, self)
    self.distance += 1

class NoOpVehicle(Vehicle):

  def move(self):
    return 0

class HumanVehicle(Vehicle):

  def setGraphics(self, graphics):
    self.graphics = graphics

  def move(self):
    if self.graphics.currentKey is not None:
      key = self.graphics.currentKey - 48
      if key >= 1 and key <= 9:
        return key - 5

    return 0

class Model(object):

  def update(self, sensorValue, motorValue):
    pass

  def predict(self):
    raise NotImplementedError

class HTMPositionModel(Model):

  def __init__(self, sparsity=0.02, encoderResolution=0.5, tmParams=None):
    tmParams = tmParams or {}
    self.tm = MonitoredGeneralTemporalMemory(mmName="TM", **tmParams)
    self.n = self.tm.numberOfColumns()
    self.w = int(self.n * sparsity) + 1
    self.sensorEncoder = RandomDistributedScalarEncoder(encoderResolution,
                                                        w=self.w,
                                                        n=self.n)
    self.motorEncoder = RandomDistributedScalarEncoder(encoderResolution,
                                                       w=self.w,
                                                       n=self.n)

  def update(self, sensorValue, motorValue):
    sensorPattern = set(self.sensorEncoder.encode(sensorValue).nonzero()[0])
    motorPattern = set(self.motorEncoder.encode(motorValue).nonzero()[0])

    self.tm.compute(sensorPattern,
                    activeExternalCells=motorPattern,
                    formInternalConnections=True,
                    learn=True)

  def predict(self):
    print self.tm.predictedCells
    return set()  # TODO

class Graphics(object):

  def __init__(self, field, vehicle, scorer, model, size=(400, 600)):
    import pygame

    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model
    self.size = size
    self.pygame = pygame

    self.currentKey = None
    self.screen = None

    self.setup()

  def setup(self):
    self.pygame.init()
    self.screen = self.pygame.display.set_mode(self.size)

  def update(self):
    self.currentKey = None

    for event in self.pygame.event.get():
      if event.type == self.pygame.KEYDOWN:
        self.currentKey = event.key

  def render(self):
    self.renderBackground()
    self.renderRoad()
    self.renderVehicle()

    self.pygame.display.flip()

  def renderBackground(self):
    black = (0, 0, 0)
    self.screen.fill(black)

  def renderVehicle(self):
    color = (0, 255, 0)
    x = self._scale(self.vehicle.position)
    y = float(self.size[1]) / 2
    self.pygame.draw.rect(self.screen,
                          color,
                          self.pygame.Rect(x, y, 10, 10))

  def renderRoad(self):
    color = (255, 0, 0)

    for y in range(self.size[1]):
      distance = self.vehicle.distance + y - float(self.size[1]) / 2

      if distance < 0:
        continue

      position, width = self.field.road.get(distance, self.field)
      x = self._scale(float(position - width) / 2)
      w = self._scale(width)

      self.pygame.draw.rect(self.screen,
                            color,
                            self.pygame.Rect(x, self.size[1] - y, w, 2))

  def _scale(self, x):
    return float(x * self.size[0]) / self.field.width

class Plots(object):

  def __init__(self, field, vehicle, scorer, model):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model

    import matplotlib.pyplot as plt
    self.plt = plt
    # import matplotlib.cm as cm
    # self.cm = cm

    self.plt.ion()
    self.plt.show()

    self.positions = []
    self.motorValues = []
    self.scores = []

  def update(self):
    self.positions.append(self.vehicle.position)
    self.motorValues.append(self.vehicle.motorValue)
    self.scores.append(self.scorer.score)

  def render(self):
    rows = 2
    cols = 2
    self.plt.clf()

    self.plt.subplot(rows, cols, 1)
    self.plt.ylim([0, self.field.width])
    self.plt.plot(range(len(self.positions)), self.positions)

    self.plt.subplot(rows, cols, 2)
    self.plt.plot(range(len(self.motorValues)), self.motorValues)

    self.plt.subplot(rows, cols, 3)
    self.plt.plot(range(len(self.scores)), self.scores)

    self.plt.draw()

class HTMPlots(Plots):

  def render(self):
    self.plt.figure(1)
    super(HTMPlots, self).render()

    # self.plt.figure(2)
    # self.model.tm.mmGetCellActivityPlot()

    self.plt.figure(2)
    self.plt.clf()
    rows = 3
    cols = 1

    data = self.model.tm.mmGetTraceUnpredictedActiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 1)
    self.plt.ylabel("Unpredicted active columns")
    self.plt.plot(range(len(data)), data)

    data = self.model.tm.mmGetTracePredictedActiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 2)
    self.plt.ylabel("Predicted active columns")
    self.plt.plot(range(len(data)), data)

    data = self.model.tm.mmGetTracePredictedInactiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 3)
    self.plt.ylabel("Predicted inactive columns")
    self.plt.plot(range(len(data)), data)

class Logs(object):

  def __init__(self, field, vehicle, scorer, model):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model

  def log(self):
    print self.scorer.score

class Scorer(object):

  def __init__(self, field, vehicle):
    self.field = field
    self.vehicle = vehicle

    self.score = 0
    self.scoreDelta = 0

  def getChange(self):
    raise NotImplementedError

  def update(self):
    self.scoreDelta = self.getChange()
    self.score += self.scoreDelta

class StayOnRoadScorer(Scorer):

  def getChange(self):
    position, width = self.field.road.get(self.vehicle.distance, self.field)
    left = float(position - width) / 2
    right = left + width

    if self.vehicle.position >= left and self.vehicle.position <= right:
      return 1
    else:
      return -1

class Game(object):

  def __init__(self, field, vehicle, scorer, model,
               logs=None, plots=None, graphics=None):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model
    self.logs = logs
    self.plots = plots
    self.graphics = graphics
    self.logs = logs
    self.graphics = graphics

  def run(self):
    i = 0
    while True:
      if self.logs is not None:
        self.logs.log()

      if self.plots is not None:
        self.plots.update()

        if i % PLOT_EVERY == 0:
          self.plots.render()

      if self.graphics is not None:
        self.graphics.update()
        self.graphics.render()

      self.vehicle.tick()
      self.scorer.update()
      self.model.update(self.vehicle.sensorValue, self.vehicle.motorValue)

      i += 1
<commit_msg>Added overlap plot<commit_after>import random

import numpy

from sensorimotor.general_temporal_memory import GeneralTemporalMemory
from nupic.research.monitor_mixin.temporal_memory_monitor_mixin import (
  TemporalMemoryMonitorMixin)
class MonitoredGeneralTemporalMemory(TemporalMemoryMonitorMixin,
                                     GeneralTemporalMemory): pass

from nupic.encoders.random_distributed_scalar import (
  RandomDistributedScalarEncoder)

PLOT_EVERY = 25

class Road(object):

  def get(self, distance, field):
    raise NotImplementedError

class StraightRoad(Road):

  def get(self, distance, field):
    return (float(field.width) / 2, 25)

class ZigZagRoad(Road):

  def __init__(self, width=25, zigZagEvery=60):
    super(ZigZagRoad, self).__init__()

    self.width = width
    self.zigZagEvery = zigZagEvery

  def get(self, distance, field):
    zig = int(distance / self.zigZagEvery)
    zag = int((distance + self.zigZagEvery) / self.zigZagEvery)

    zigDistance = distance % self.zigZagEvery

    numpy.random.seed(zig)
    zigPosition = numpy.random.randint(0, field.width)
    numpy.random.seed(zag)
    zagPosition = numpy.random.randint(0, field.width)
    separation = zagPosition - zigPosition

    zigPercent = float(zigDistance) / self.zigZagEvery
    position = zigPosition + (separation * zigPercent)

    return (position, self.width)

  def containsVehicle(self, vehicle):
    if vehicle.position > 50:
      return True

    return False

class Field(object):

  def __init__(self, road, width=100):
    self.width = width
    self.road = road

class Sensor(object):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def sense(self, field, vehicle):
    raise NotImplementedError

class NoOpSensor(Sensor):

  def sense(self, field, vehicle):
    return None

class PositionSensor(Sensor):

  def sense(self, field, vehicle):
    return vehicle.position

class Motor(object):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def move(self, motorValue, vehicle):
    raise NotImplementedError

class AccelerationMotor(Motor):

  def __init__(self, frictionCoefficient=0.1, noise=(0.0, 0.0)):
    self.frictionCoefficient = frictionCoefficient
    self.noise = noise

  def move(self, motorValue, vehicle):
    acceleration = motorValue + random.gauss(*self.noise)
    friction = vehicle.velocity * self.frictionCoefficient
    vehicle.velocity += acceleration - friction
    vehicle.position += vehicle.velocity
    vehicle.position = vehicle.position % vehicle.field.width

class PositionMotor(Motor):

  def __init__(self, noise=(0.0, 0.0)):
    self.noise = noise

  def move(self, motorValue, vehicle):
    vehicle.position += motorValue + random.gauss(*self.noise)

class Vehicle(object):

  def __init__(self, field, sensor, motor):
    self.field = field
    self.sensor = sensor
    self.motor = motor

    self.position = 0
    self.distance = 0
    self.velocity = 0

    self.sensorValue = None
    self.motorValue = None

  def move(self):
    raise NotImplementedError()

  def tick(self):
    self.sensorValue = self.sensor.sense(self.field, self)
    self.motorValue = self.move()
    self.motor.move(self.motorValue, self)
    self.distance += 1

class NoOpVehicle(Vehicle):

  def move(self):
    return 0

class HumanVehicle(Vehicle):

  def setGraphics(self, graphics):
    self.graphics = graphics

  def move(self):
    if self.graphics.currentKey is not None:
      key = self.graphics.currentKey - 48
      if key >= 1 and key <= 9:
        return key - 5

    return 0

class Model(object):

  def update(self, sensorValue, motorValue):
    pass

  def predict(self):
    raise NotImplementedError

class HTMPositionModel(Model):

  def __init__(self, sparsity=0.02, encoderResolution=0.05, tmParams=None):
    tmParams = tmParams or {}
    self.tm = MonitoredGeneralTemporalMemory(mmName="TM", **tmParams)
    self.n = self.tm.numberOfColumns()
    self.w = int(self.n * sparsity) + 1
    self.sensorEncoder = RandomDistributedScalarEncoder(encoderResolution,
                                                        w=self.w,
                                                        n=self.n)
    self.motorEncoder = RandomDistributedScalarEncoder(encoderResolution,
                                                       w=self.w,
                                                       n=self.n)

  def update(self, sensorValue, motorValue):
    sensorPattern = set(self.sensorEncoder.encode(sensorValue).nonzero()[0])
    motorPattern = set(self.motorEncoder.encode(motorValue).nonzero()[0])

    self.tm.compute(sensorPattern,
                    activeExternalCells=motorPattern,
                    formInternalConnections=True,
                    learn=True)

  def predict(self):
    print self.tm.predictedCells
    return set()  # TODO

class Graphics(object):

  def __init__(self, field, vehicle, scorer, model, size=(400, 600)):
    import pygame

    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model
    self.size = size
    self.pygame = pygame

    self.currentKey = None
    self.screen = None

    self.setup()

  def setup(self):
    self.pygame.init()
    self.screen = self.pygame.display.set_mode(self.size)

  def update(self):
    self.currentKey = None

    for event in self.pygame.event.get():
      if event.type == self.pygame.KEYDOWN:
        self.currentKey = event.key

  def render(self):
    self.renderBackground()
    self.renderRoad()
    self.renderVehicle()

    self.pygame.display.flip()

  def renderBackground(self):
    black = (0, 0, 0)
    self.screen.fill(black)

  def renderVehicle(self):
    color = (0, 255, 0)
    x = self._scale(self.vehicle.position)
    y = float(self.size[1]) / 2
    self.pygame.draw.rect(self.screen,
                          color,
                          self.pygame.Rect(x, y, 10, 10))

  def renderRoad(self):
    color = (255, 0, 0)

    for y in range(self.size[1]):
      distance = self.vehicle.distance + y - float(self.size[1]) / 2

      if distance < 0:
        continue

      position, width = self.field.road.get(distance, self.field)
      x = self._scale(float(position - width) / 2)
      w = self._scale(width)

      self.pygame.draw.rect(self.screen,
                            color,
                            self.pygame.Rect(x, self.size[1] - y, w, 2))

  def _scale(self, x):
    return float(x * self.size[0]) / self.field.width

class Plots(object):

  def __init__(self, field, vehicle, scorer, model):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model

    import matplotlib.pyplot as plt
    self.plt = plt
    # import matplotlib.cm as cm
    # self.cm = cm

    self.plt.ion()
    self.plt.show()

    self.positions = []
    self.motorValues = []
    self.scores = []

  def update(self):
    self.positions.append(self.vehicle.position)
    self.motorValues.append(self.vehicle.motorValue)
    self.scores.append(self.scorer.score)

  def render(self):
    rows = 2
    cols = 2
    self.plt.clf()

    self.plt.subplot(rows, cols, 1)
    self.plt.ylim([0, self.field.width])
    self.plt.plot(range(len(self.positions)), self.positions)

    self.plt.subplot(rows, cols, 2)
    self.plt.plot(range(len(self.motorValues)), self.motorValues)

    self.plt.subplot(rows, cols, 3)
    self.plt.plot(range(len(self.scores)), self.scores)

    self.plt.draw()

class HTMPlots(Plots):

  def render(self):
    self.plt.figure(1)
    super(HTMPlots, self).render()

    # self.plt.figure(2)
    # self.model.tm.mmGetCellActivityPlot()

    self.plt.figure(2)
    self.plt.clf()
    rows = 4
    cols = 1

    data = self.model.tm.mmGetTraceActiveColumns().data
    overlaps = [len(a & b) for a, b in zip(data[:-1], data[1:])]
    self.plt.subplot(rows, cols, 1)
    self.plt.ylabel("Active columns overlap with t-1")
    self.plt.plot(range(len(overlaps)), overlaps)

    data = self.model.tm.mmGetTraceUnpredictedActiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 2)
    self.plt.ylabel("Unpredicted active columns")
    self.plt.plot(range(len(data)), data)

    data = self.model.tm.mmGetTracePredictedActiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 3)
    self.plt.ylabel("Predicted active columns")
    self.plt.plot(range(len(data)), data)

    data = self.model.tm.mmGetTracePredictedInactiveColumns().makeCountsTrace().data
    self.plt.subplot(rows, cols, 4)
    self.plt.ylabel("Predicted inactive columns")
    self.plt.plot(range(len(data)), data)

class Logs(object):

  def __init__(self, field, vehicle, scorer, model):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model

  def log(self):
    print self.scorer.score

class Scorer(object):

  def __init__(self, field, vehicle):
    self.field = field
    self.vehicle = vehicle

    self.score = 0
    self.scoreDelta = 0

  def getChange(self):
    raise NotImplementedError

  def update(self):
    self.scoreDelta = self.getChange()
    self.score += self.scoreDelta

class StayOnRoadScorer(Scorer):

  def getChange(self):
    position, width = self.field.road.get(self.vehicle.distance, self.field)
    left = float(position - width) / 2
    right = left + width

    if self.vehicle.position >= left and self.vehicle.position <= right:
      return 1
    else:
      return -1

class Game(object):

  def __init__(self, field, vehicle, scorer, model,
               logs=None, plots=None, graphics=None):
    self.field = field
    self.vehicle = vehicle
    self.scorer = scorer
    self.model = model
    self.logs = logs
    self.plots = plots
    self.graphics = graphics
    self.logs = logs
    self.graphics = graphics

  def run(self):
    i = 0
    while True:
      if self.logs is not None:
        self.logs.log()

      if self.plots is not None:
        self.plots.update()

        if i % PLOT_EVERY == 0:
          self.plots.render()

      if self.graphics is not None:
        self.graphics.update()
        self.graphics.render()

      self.vehicle.tick()
      self.scorer.update()
      self.model.update(self.vehicle.sensorValue, self.vehicle.motorValue)

      i += 1
<|endoftext|>


<commit_before>from logging import getLogger

from django.core.paginator import Paginator
from django.shortcuts import render
from django.utils.datastructures import SortedDict
from django.conf import settings
from django.utils.functional import curry
from django.http import HttpResponseNotFound

from redis.exceptions import ResponseError

from .utils import LazySlicingIterable

logger = getLogger(__name__)

REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)

def safeint(value):
    try:
        return int(value)
    except ValueError:
        return value

def _fixup_pair(pair):
    a, b = pair
    return a, safeint(b)

LENGTH_GETTERS = {
    b'list': lambda conn, key: conn.llen(key),
    b'string': lambda conn, key: conn.strlen(key),
    b'set': lambda conn, key: conn.scard(key),
    b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
    b'hash': lambda conn, key: conn.hlen(key),
}

def _get_key_info(conn, key):
    try:
        obj_type = conn.type(key)
        pipe = conn.pipeline()
        pipe.execute_command('DEBUG', 'OBJECT', key)
        LENGTH_GETTERS[obj_type](pipe, key)
        pipe.ttl(key)
        details, obj_length, obj_ttl = pipe.execute()
        return {
            'type': obj_type,
            'name': key,
            'details': details if isinstance(details, dict) else dict(
                _fixup_pair(i.split(b':')) for i in details.split() if b':' in i
            ),
            'length': obj_length,
            'ttl': obj_ttl,
        }
    except ResponseError as exc:
        logger.exception("Failed to get details for key %r", key)
        return {
            'type': "n/a",
            'length': "n/a",
            'name': key,
            'details': {},
            'error': str(exc),
            'ttl': "n/a",
        }

VALUE_GETTERS = {
    b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.lrange(key, start, end))],
    b'string': lambda conn, key, *args: [('string', conn.get(key))],
    b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
    b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.zrange(key, start, end))],
    b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
    b'n/a': lambda conn, key, *args: (),
}

def _get_key_details(conn, db, key, page):
    conn.execute_command('SELECT', db)
    details = _get_key_info(conn, key)
    details['db'] = db
    if details['type'] in ('list', 'zset'):
        details['data'] = Paginator(
            LazySlicingIterable(
                lambda: details['length'],
                curry(VALUE_GETTERS[details['type']], conn, key)
            ),
            REDISBOARD_ITEMS_PER_PAGE
        ).page(page)
    else:
        details['data'] = VALUE_GETTERS[details['type']](conn, key)

    return details

def _get_db_summary(server, db):
    server.connection.execute_command('SELECT', db)
    pipe = server.connection.pipeline()

    pipe.dbsize()
    for i in range(server.sampling_threshold):
        pipe.randomkey()

    results = pipe.execute()
    size = results.pop(0)
    keys = sorted(set(results))

    pipe = server.connection.pipeline()
    for key in keys:
        pipe.execute_command('DEBUG', 'OBJECT', key)
        pipe.ttl(key)

    total_memory = 0
    volatile_memory = 0
    persistent_memory = 0
    total_keys = 0
    volatile_keys = 0
    persistent_keys = 0
    results = pipe.execute()
    for key, details, ttl in zip(keys, results[::2], results[1::2]):
        if not isinstance(details, dict):
            details = dict(_fixup_pair(i.split(b':'))
                           for i in details.split() if b':' in i)

        length = details[b'serializedlength'] + len(key)

        if ttl:
            persistent_memory += length
            persistent_keys += 1
        else:
            volatile_memory += length
            volatile_keys += 1
        total_memory += length
        total_keys += 1

    if total_keys:
        total_memory = (total_memory / total_keys) * size
    else:
        total_memory = 0

    if persistent_keys:
        persistent_memory = (persistent_memory / persistent_keys) * size
    else:
        persistent_memory = 0

    if volatile_keys:
        volatile_memory = (volatile_memory / volatile_keys) * size
    else:
        volatile_memory = 0

    return dict(
        size=size,
        total_memory=total_memory,
        volatile_memory=volatile_memory,
        persistent_memory=persistent_memory,
    )

def _get_db_details(server, db):
    conn = server.connection
    conn.execute_command('SELECT', db)
    size = conn.dbsize()

    key_details = {}
    if size > server.sampling_threshold:
        sampling = True
        pipe = conn.pipeline()
        for _ in xrange(server.sampling_size):
            pipe.randomkey()

        for key in set(pipe.execute()):
            key_details[key] = _get_key_info(conn, key)

    else:
        sampling = False
        for key in conn.keys():
            key_details[key] = _get_key_info(conn, key)

    return dict(
        keys=key_details,
        sampling=sampling,
    )

def inspect(request, server):
    stats = server.stats
    conn = server.connection
    database_details = SortedDict()
    key_details = None

    if stats['status'] == 'UP':
        if 'key' in request.GET:
            key = request.GET['key']
            db = request.GET.get('db', 0)
            page = request.GET.get('page', 1)
            key_details = _get_key_details(conn, db, key, page)
        else:
            databases = sorted(name[2:] for name in conn.info()
                               if name.startswith('db'))
            total_size = 0
            for db in databases:
                database_details[db] = summary = _get_db_summary(server, db)
                total_size += summary['size']
            if total_size < server.sampling_threshold:
                for db in databases:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
            elif 'db' in request.GET:
                db = request.GET['db']
                if db in database_details:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
                else:
                    return HttpResponseNotFound("Unknown database.")

    return render(request, "redisboard/inspect.html", {
        'databases': database_details,
        'key_details': key_details,
        'original': server,
        'stats': stats,
        'app_label': 'redisboard',
    })
<commit_msg>Fix a py3 issue.<commit_after>from logging import getLogger

from django.core.paginator import Paginator
from django.shortcuts import render
from django.utils.datastructures import SortedDict
from django.conf import settings
from django.utils.functional import curry
from django.http import HttpResponseNotFound

from redis.exceptions import ResponseError

from .utils import LazySlicingIterable
from .utils import PY3

logger = getLogger(__name__)

REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)

def safeint(value):
    try:
        return int(value)
    except ValueError:
        return value

def _fixup_pair(pair):
    a, b = pair
    return a, safeint(b)

LENGTH_GETTERS = {
    b'list': lambda conn, key: conn.llen(key),
    b'string': lambda conn, key: conn.strlen(key),
    b'set': lambda conn, key: conn.scard(key),
    b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
    b'hash': lambda conn, key: conn.hlen(key),
}

def _get_key_info(conn, key):
    try:
        obj_type = conn.type(key)
        pipe = conn.pipeline()
        pipe.execute_command('DEBUG', 'OBJECT', key)
        LENGTH_GETTERS[obj_type](pipe, key)
        pipe.ttl(key)
        details, obj_length, obj_ttl = pipe.execute()
        return {
            'type': obj_type,
            'name': key,
            'details': details if isinstance(details, dict) else dict(
                _fixup_pair(i.split(b':')) for i in details.split() if b':' in i
            ),
            'length': obj_length,
            'ttl': obj_ttl,
        }
    except ResponseError as exc:
        logger.exception("Failed to get details for key %r", key)
        return {
            'type': "n/a",
            'length': "n/a",
            'name': key,
            'details': {},
            'error': str(exc),
            'ttl': "n/a",
        }

VALUE_GETTERS = {
    b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.lrange(key, start, end))],
    b'string': lambda conn, key, *args: [('string', conn.get(key))],
    b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
    b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.zrange(key, start, end))],
    b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
    b'n/a': lambda conn, key, *args: (),
}

def _get_key_details(conn, db, key, page):
    conn.execute_command('SELECT', db)
    details = _get_key_info(conn, key)
    details['db'] = db
    if details['type'] in ('list', 'zset'):
        details['data'] = Paginator(
            LazySlicingIterable(
                lambda: details['length'],
                curry(VALUE_GETTERS[details['type']], conn, key)
            ),
            REDISBOARD_ITEMS_PER_PAGE
        ).page(page)
    else:
        details['data'] = VALUE_GETTERS[details['type']](conn, key)

    return details

def _get_db_summary(server, db):
    server.connection.execute_command('SELECT', db)
    pipe = server.connection.pipeline()

    pipe.dbsize()
    for i in range(server.sampling_threshold):
        pipe.randomkey()

    results = pipe.execute()
    size = results.pop(0)
    keys = sorted(set(results))

    pipe = server.connection.pipeline()
    for key in keys:
        pipe.execute_command('DEBUG', 'OBJECT', key)
        pipe.ttl(key)

    total_memory = 0
    volatile_memory = 0
    persistent_memory = 0
    total_keys = 0
    volatile_keys = 0
    persistent_keys = 0
    results = pipe.execute()
    for key, details, ttl in zip(keys, results[::2], results[1::2]):
        if not isinstance(details, dict):
            details = dict(_fixup_pair(i.split(b':'))
                           for i in details.split() if b':' in i)

        length = details[b'serializedlength'] + len(key)

        if ttl:
            persistent_memory += length
            persistent_keys += 1
        else:
            volatile_memory += length
            volatile_keys += 1
        total_memory += length
        total_keys += 1

    if total_keys:
        total_memory = (total_memory / total_keys) * size
    else:
        total_memory = 0

    if persistent_keys:
        persistent_memory = (persistent_memory / persistent_keys) * size
    else:
        persistent_memory = 0

    if volatile_keys:
        volatile_memory = (volatile_memory / volatile_keys) * size
    else:
        volatile_memory = 0

    return dict(
        size=size,
        total_memory=total_memory,
        volatile_memory=volatile_memory,
        persistent_memory=persistent_memory,
    )

def _get_db_details(server, db):
    conn = server.connection
    conn.execute_command('SELECT', db)
    size = conn.dbsize()

    key_details = {}
    if size > server.sampling_threshold:
        sampling = True
        pipe = conn.pipeline()
        for _ in (range if PY3 else xrange)(server.sampling_size):
            pipe.randomkey()

        for key in set(pipe.execute()):
            key_details[key] = _get_key_info(conn, key)

    else:
        sampling = False
        for key in conn.keys():
            key_details[key] = _get_key_info(conn, key)

    return dict(
        keys=key_details,
        sampling=sampling,
    )

def inspect(request, server):
    stats = server.stats
    conn = server.connection
    database_details = SortedDict()
    key_details = None

    if stats['status'] == 'UP':
        if 'key' in request.GET:
            key = request.GET['key']
            db = request.GET.get('db', 0)
            page = request.GET.get('page', 1)
            key_details = _get_key_details(conn, db, key, page)
        else:
            databases = sorted(name[2:] for name in conn.info()
                               if name.startswith('db'))
            total_size = 0
            for db in databases:
                database_details[db] = summary = _get_db_summary(server, db)
                total_size += summary['size']
            if total_size < server.sampling_threshold:
                for db in databases:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
            elif 'db' in request.GET:
                db = request.GET['db']
                if db in database_details:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
                else:
                    return HttpResponseNotFound("Unknown database.")

    return render(request, "redisboard/inspect.html", {
        'databases': database_details,
        'key_details': key_details,
        'original': server,
        'stats': stats,
        'app_label': 'redisboard',
    })
<|endoftext|>


<commit_before>from rez.config import config
from rez.resolved_context import ResolvedContext
from rez.packages_ import get_latest_package_from_string, Variant
from rez.exceptions import PackageNotFoundError, PackageTestError
from rez.utils.colorize import heading, Printer
from pipes import quote
import subprocess
import time
import sys

class PackageTestRunner(object):
    """Object for running a package's tests.

    This runs the tests listed in the package's "tests" attribute.

    An example tests entry in a package.py might look like this:

        tests = {
            "unit": "python -m unittest -s {root}/tests",
            "CI": {
                "command": "python {root}/ci_tests/main.py",
                "requires": ["maya-2017"],
                "replace": True
            }
        }

    By default tests are run in an environment containing the current package.

    If a test entry is just a string, then it is treated as the test
    command. If a dict, the "command" string is the command, and the "requires"
    list is added to the test env.

    Command strings automatically expand references such as '{root}', much
    as happens in a *commands* function.

    Commands can also be a list - in this case, the test process is launched
    directly, rather than interpreted via a shell.

    TODO FIXME: Currently a test will not be run over the variants of a
    package. This is because there is no reliable way to resolve to a specific
    variant's env - we can influence the variant chosen, knowing how the
    variant selection mode works, but this is not a guarantee and it would
    be error-prone and complicated to do it this way. For reasons beyond
    package testing, we want to be able to explicitly specify a variant to
    resolve to anyway, so this will be fixed in a separate feature. Once that
    is available, this code will be updated to iterate over a package's
    variants and run tests in each.
    """
    def __init__(self, package_request, use_current_env=False,
                 extra_package_requests=None, package_paths=None, stdout=None,
                 stderr=None, verbose=False, **context_kwargs):
        """Create a package tester.

        Args:
            package_request (str or `PackageRequest`): The package to test.
            use_current_env (bool): If True, run the test directly in the current
                rez-resolved environment, if there is one, and if it contains
                packages that meet the test's requirements.
            extra_package_requests (list of str or `PackageRequest`): Extra
                requests, these are appended to the test environment.
            package_paths: List of paths to search for pkgs, defaults to
                config.packages_path.
            stdout (file-like object): Defaults to sys.stdout.
            stderr (file-like object): Defaults to sys.stderr.
            verbose (bool): Verbose mode.
            context_kwargs: Extra arguments which are passed to the
                `ResolvedContext` instances used to run the tests within.
                Ignored if `use_current_env` is True.
        """
        self.package_request = package_request
        self.use_current_env = use_current_env
        self.extra_package_requests = extra_package_requests
        self.stdout = stdout or sys.stdout
        self.stderr = stderr or sys.stderr
        self.verbose = verbose
        self.context_kwargs = context_kwargs

        self.package_paths = (config.packages_path if package_paths is None
                              else package_paths)

        self.package = None
        self.contexts = {}

        # use a common timestamp across all tests - this ensures that tests
        # don't pick up new packages halfway through (ie from one test to another)
        self.timestamp = int(time.time())

        if use_current_env:
            raise NotImplementedError

    def get_package(self):
        """Get the target package.

        Returns:
            `Package`: Package to run tests on.
        """
        if self.package is not None:
            return self.package

        if self.use_current_env:
            pass
        else:
            package = get_latest_package_from_string(str(self.package_request),
                                                     self.package_paths)
            if package is None:
                raise PackageNotFoundError("Could not find package to test: %s"
                                           % str(self.package_request))

        self.package = package
        return self.package

    def get_test_names(self):
        """Get the names of tests in this package.

        Returns:
            List of str: Test names.
        """
        package = self.get_package()
        return sorted((package.tests or {}).keys())

    def run_test(self, test_name):
        """Run a test.

        Runs the test in its correct environment. Note that if tests share the
        same requirements, the contexts will be reused.

        TODO: If the package had variants, the test will be run for each
        variant.

        Returns:
            int: Returncode - zero if all test(s) passed, otherwise the return
                code of the failed test.
        """
        def print_header(txt, *nargs):
            pr = Printer(sys.stdout)
            pr(txt % nargs, heading)

        package = self.get_package()

        if test_name not in self.get_test_names():
            raise PackageTestError("Test '%s' not found in package %s"
                                   % (test_name, package.uri))

        if self.use_current_env:
            return self._run_test_in_current_env(test_name)

        for variant in package.iter_variants():

            # get test info for this variant. If None, that just means that this
            # variant doesn't provide this test. That's ok - 'tests' might be
            # implemented as a late function attribute that provides some tests
            # for some variants and not others
            #
            test_info = self._get_test_info(test_name, variant)
            if not test_info:
                continue

            command = test_info["command"]
            requires = test_info["requires"]

            # expand refs like {root} in commands
            if isinstance(command, basestring):
                command = variant.format(command)
            else:
                command = map(variant.format, command)

            # show progress
            if self.verbose:
                print_header(
                    "\nTest: %s\nPackage: %s\n%s\n",
                    test_name, variant.uri, '-' * 80)

            # create test env
            key = tuple(requires)
            context = self.contexts.get(key)

            if context is None:
                if self.verbose:
                    print_header("Resolving test environment: %s\n",
                                 ' '.join(map(quote, requires)))

                context = ResolvedContext(package_requests=requires,
                                          package_paths=self.package_paths,
                                          buf=self.stdout,
                                          timestamp=self.timestamp,
                                          **self.context_kwargs)

                if not context.success:
                    context.print_info(buf=self.stderr)
                    raise PackageTestError(
                        "Cannot run test '%s' of package %s: the environment "
                        "failed to resolve" % (test_name, variant.uri))

                self.contexts[key] = context

            # run the test in the context
            if self.verbose:
                context.print_info(self.stdout)

                if isinstance(command, basestring):
                    cmd_str = command
                else:
                    cmd_str = ' '.join(map(quote, command))

                print_header("\nRunning test command: %s\n" % cmd_str)

            retcode, _, _ = context.execute_shell(
                command=command,
                stdout=self.stdout,
                stderr=self.stderr,
                block=True)

            if retcode:
                return retcode

            # TODO FIXME we don't iterate over all variants yet, because we
            # can't reliably do that (see class docstring)
            break

        return 0  # success

    def _get_test_info(self, test_name, variant):
        tests_dict = variant.tests or {}
        test_entry = tests_dict.get(test_name)

        if not test_entry:
            return None

        if not isinstance(test_entry, dict):
            test_entry = {
                "command": test_entry
            }

        # construct env request
        requires = []

        if len(variant.version):
            req = "%s==%s" % (variant.name, str(variant.version))
            requires.append(req)
        else:
            requires.append(variant.name)

        reqs = test_entry.get("requires") or []
        requires.extend(map(str, reqs))

        if self.extra_package_requests:
            reqs = map(str, self.extra_package_requests)
            requires.extend(reqs)

        return {
            "command": test_entry["command"],
            "requires": requires
        }
<commit_msg>package_test.py: fix rez-test header command with %<commit_after>from rez.config import config
from rez.resolved_context import ResolvedContext
from rez.packages_ import get_latest_package_from_string, Variant
from rez.exceptions import PackageNotFoundError, PackageTestError
from rez.utils.colorize import heading, Printer
from pipes import quote
import subprocess
import time
import sys

class PackageTestRunner(object):
    """Object for running a package's tests.

    This runs the tests listed in the package's "tests" attribute.

    An example tests entry in a package.py might look like this:

        tests = {
            "unit": "python -m unittest -s {root}/tests",
            "CI": {
                "command": "python {root}/ci_tests/main.py",
                "requires": ["maya-2017"],
                "replace": True
            }
        }

    By default tests are run in an environment containing the current package.

    If a test entry is just a string, then it is treated as the test
    command. If a dict, the "command" string is the command, and the "requires"
    list is added to the test env.

    Command strings automatically expand references such as '{root}', much
    as happens in a *commands* function.

    Commands can also be a list - in this case, the test process is launched
    directly, rather than interpreted via a shell.

    TODO FIXME: Currently a test will not be run over the variants of a
    package. This is because there is no reliable way to resolve to a specific
    variant's env - we can influence the variant chosen, knowing how the
    variant selection mode works, but this is not a guarantee and it would
    be error-prone and complicated to do it this way. For reasons beyond
    package testing, we want to be able to explicitly specify a variant to
    resolve to anyway, so this will be fixed in a separate feature. Once that
    is available, this code will be updated to iterate over a package's
    variants and run tests in each.
    """
    def __init__(self, package_request, use_current_env=False,
                 extra_package_requests=None, package_paths=None, stdout=None,
                 stderr=None, verbose=False, **context_kwargs):
        """Create a package tester.

        Args:
            package_request (str or `PackageRequest`): The package to test.
            use_current_env (bool): If True, run the test directly in the current
                rez-resolved environment, if there is one, and if it contains
                packages that meet the test's requirements.
            extra_package_requests (list of str or `PackageRequest`): Extra
                requests, these are appended to the test environment.
            package_paths: List of paths to search for pkgs, defaults to
                config.packages_path.
            stdout (file-like object): Defaults to sys.stdout.
            stderr (file-like object): Defaults to sys.stderr.
            verbose (bool): Verbose mode.
            context_kwargs: Extra arguments which are passed to the
                `ResolvedContext` instances used to run the tests within.
                Ignored if `use_current_env` is True.
        """
        self.package_request = package_request
        self.use_current_env = use_current_env
        self.extra_package_requests = extra_package_requests
        self.stdout = stdout or sys.stdout
        self.stderr = stderr or sys.stderr
        self.verbose = verbose
        self.context_kwargs = context_kwargs

        self.package_paths = (config.packages_path if package_paths is None
                              else package_paths)

        self.package = None
        self.contexts = {}

        # use a common timestamp across all tests - this ensures that tests
        # don't pick up new packages halfway through (ie from one test to another)
        self.timestamp = int(time.time())

        if use_current_env:
            raise NotImplementedError

    def get_package(self):
        """Get the target package.

        Returns:
            `Package`: Package to run tests on.
        """
        if self.package is not None:
            return self.package

        if self.use_current_env:
            pass
        else:
            package = get_latest_package_from_string(str(self.package_request),
                                                     self.package_paths)
            if package is None:
                raise PackageNotFoundError("Could not find package to test: %s"
                                           % str(self.package_request))

        self.package = package
        return self.package

    def get_test_names(self):
        """Get the names of tests in this package.

        Returns:
            List of str: Test names.
        """
        package = self.get_package()
        return sorted((package.tests or {}).keys())

    def run_test(self, test_name):
        """Run a test.

        Runs the test in its correct environment. Note that if tests share the
        same requirements, the contexts will be reused.

        TODO: If the package had variants, the test will be run for each
        variant.

        Returns:
            int: Returncode - zero if all test(s) passed, otherwise the return
                code of the failed test.
        """
        def print_header(txt, *nargs):
            pr = Printer(sys.stdout)
            pr(txt % nargs, heading)

        package = self.get_package()

        if test_name not in self.get_test_names():
            raise PackageTestError("Test '%s' not found in package %s"
                                   % (test_name, package.uri))

        if self.use_current_env:
            return self._run_test_in_current_env(test_name)

        for variant in package.iter_variants():

            # get test info for this variant. If None, that just means that this
            # variant doesn't provide this test. That's ok - 'tests' might be
            # implemented as a late function attribute that provides some tests
            # for some variants and not others
            #
            test_info = self._get_test_info(test_name, variant)
            if not test_info:
                continue

            command = test_info["command"]
            requires = test_info["requires"]

            # expand refs like {root} in commands
            if isinstance(command, basestring):
                command = variant.format(command)
            else:
                command = map(variant.format, command)

            # show progress
            if self.verbose:
                print_header(
                    "\nTest: %s\nPackage: %s\n%s\n",
                    test_name, variant.uri, '-' * 80)

            # create test env
            key = tuple(requires)
            context = self.contexts.get(key)

            if context is None:
                if self.verbose:
                    print_header("Resolving test environment: %s\n",
                                 ' '.join(map(quote, requires)))

                context = ResolvedContext(package_requests=requires,
                                          package_paths=self.package_paths,
                                          buf=self.stdout,
                                          timestamp=self.timestamp,
                                          **self.context_kwargs)

                if not context.success:
                    context.print_info(buf=self.stderr)
                    raise PackageTestError(
                        "Cannot run test '%s' of package %s: the environment "
                        "failed to resolve" % (test_name, variant.uri))

                self.contexts[key] = context

            # run the test in the context
            if self.verbose:
                context.print_info(self.stdout)

                if isinstance(command, basestring):
                    cmd_str = command
                else:
                    cmd_str = ' '.join(map(quote, command))

                print_header("\nRunning test command: %s\n", cmd_str)

            retcode, _, _ = context.execute_shell(
                command=command,
                stdout=self.stdout,
                stderr=self.stderr,
                block=True)

            if retcode:
                return retcode

            # TODO FIXME we don't iterate over all variants yet, because we
            # can't reliably do that (see class docstring)
            break

        return 0  # success

    def _get_test_info(self, test_name, variant):
        tests_dict = variant.tests or {}
        test_entry = tests_dict.get(test_name)

        if not test_entry:
            return None

        if not isinstance(test_entry, dict):
            test_entry = {
                "command": test_entry
            }

        # construct env request
        requires = []

        if len(variant.version):
            req = "%s==%s" % (variant.name, str(variant.version))
            requires.append(req)
        else:
            requires.append(variant.name)

        reqs = test_entry.get("requires") or []
        requires.extend(map(str, reqs))

        if self.extra_package_requests:
            reqs = map(str, self.extra_package_requests)
            requires.extend(reqs)

        return {
            "command": test_entry["command"],
            "requires": requires
        }
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import copy
import re

def split_lines(stdout):
    # we will skip the big ascii header
    lines = stdout.split("\n")
    for index, line in enumerate(lines):
        if  "| Started:" in line:
            return lines[index+1:]
    raise Exception("wpscan report does not contain proper header")

WORDPRESS = {
    "version": "",
    "is_multi_site": False,
    "is_outdated": False,
    "readme_exists": True,
    "theme": "",
    "vulnerabilities": []
}
WORDPRESS_VULN = {
    "title": "",
    "references": [],
    "fixed_since": ""
}

PLUGIN = {
    "name": "",
    "vulnerabilities": {}
}
THEME = {
    "name": "",
    "vulnerabilities": {}
}
PLUGIN_OR_THEME_VULN = {
    "title": "",
    "references": [],
    "fixed_since": ""
}

USER = {
    "id": "",
    "login": "",
    "name": "",
    "password": ""
}
# report structure looks like this https://github.com/yeukhon/minion-wpscan-plugin/issues/1

def is_single_statement(next_line, next_second_line):
    watch_list = ("[!]", "[+]")
    for criminal in watch_list:
        if next_line.startswith(criminal) or next_second_line.startswith(criminal):
            return True
    return False
def _split(line, delim):
    splits = line.split(delim, 1)
    if len(splits) == 2:
        return splits[0].strip(), splits[1].strip()
    else:
        return None, None

with open("/home/vagrant/wpscan10", "r") as f:
    stdout = f.read()

def is_readme_exists(lines):
    for line in lines:
        if "readme.html" in line:
            return True
    return False

def get_version(lines):
    for line in lines:
        if "WordPress version" in line:
            r = re.compile("\d.\d.\d")
            return r.search(line).group()
    return "unknown"

def get_wp_vuln(lines):
    wp_vuln = None
    vuln_list = []
    for line in lines:
        if "identified from the version number" in line:
            # split the whole block on " | "or " | * " starting symbol
            _lines = filter(None, re.split("\||\\s*\|\\s*\*", line))
            # the end result is _lines[0] == title and the rest groups of title,references
            for line in _lines[1:]:
                label, value = _split(line, ":")
                if label == "Title":
                    wp_vuln = copy.deepcopy(WORDPRESS_VULN)
                    wp_vuln["title"] = value
                    vuln_list.append(wp_vuln)
                elif label == "Reference":
                    wp_vuln["references"].append(value)
                elif label == "Fixed in":
                    wp_vuln["fixed_since"] = value
    return vuln_list

def get_wp_theme_in_use(lines):
    for line in lines:
        if "WordPress theme in use" in line:
            r = re.compile("WordPress theme in use: (.+)")
            name = r.findall(line)[-1]
            return name
    return "unknown"

lines = re.split("\[\+\]|\[\!\]", stdout)
import pprint

pprint.pprint(get_wp_vuln(lines))
print "version: ", get_version(lines)
print "readme: ", is_readme_exists(lines)
print "theme in use: ", get_wp_theme_in_use(lines)
<commit_msg>Expose plugins detected. See #1.<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import copy
import re

def split_lines(stdout):
    # we will skip the big ascii header
    lines = stdout.split("\n")
    for index, line in enumerate(lines):
        if  "| Started:" in line:
            return lines[index+1:]
    raise Exception("wpscan report does not contain proper header")

WORDPRESS = {
    "version": "",
    "is_multi_site": False,
    "is_outdated": False,
    "readme_exists": True,
    "theme": "",
    "vulnerabilities": []
}
WORDPRESS_VULN = {
    "title": "",
    "references": [],
    "fixed_since": ""
}

PLUGIN = {
    "name": "",
    "vulnerabilities": {}
}
THEME = {
    "name": "",
    "vulnerabilities": {}
}
PLUGIN_OR_THEME_VULN = {
    "title": "",
    "references": [],
    "fixed_since": ""
}

USER = {
    "id": "",
    "login": "",
    "name": "",
    "password": ""
}
# report structure looks like this https://github.com/yeukhon/minion-wpscan-plugin/issues/1

def is_single_statement(next_line, next_second_line):
    watch_list = ("[!]", "[+]")
    for criminal in watch_list:
        if next_line.startswith(criminal) or next_second_line.startswith(criminal):
            return True
    return False
def _split(line, delim):
    splits = line.split(delim, 1)
    if len(splits) == 2:
        return splits[0].strip(), splits[1].strip()
    else:
        return None, None

with open("/home/vagrant/wpscan12", "r") as f:
    stdout = f.read()

def is_readme_exists(lines):
    for line in lines:
        if "readme.html" in line:
            return True
    return False

def get_version(lines):
    for line in lines:
        if "WordPress version" in line:
            r = re.compile("\d.\d.\d")
            return r.search(line).group()
    return "unknown"

def get_wp_vuln(lines):
    wp_vuln = None
    vuln_list = []
    for line in lines:
        if "identified from the version number" in line:
            # split the whole block on " | "or " | * " starting symbol
            _lines = filter(None, re.split("\||\\s*\|\\s*\*", line))
            # the end result is _lines[0] == title and the rest groups of title,references
            for line in _lines[1:]:
                label, value = _split(line, ":")
                if label == "Title":
                    wp_vuln = copy.deepcopy(WORDPRESS_VULN)
                    wp_vuln["title"] = value
                    vuln_list.append(wp_vuln)
                elif label == "Reference":
                    wp_vuln["references"].append(value)
                elif label == "Fixed in":
                    wp_vuln["fixed_since"] = value
            break
    return vuln_list

def get_wp_theme_in_use(lines):
    for line in lines:
        if "WordPress theme in use" in line:
            r = re.compile("WordPress theme in use: (.+)")
            name = r.findall(line)[-1]
            return name
    return "unknown"

def get_plugins(lines):
    vuln_list = []
    plugin = {}
    plugin_vuln = {}
    for line in lines:
        if "No plugins found" in lines:
            return copy.deepcopy({})
        elif "We found" in line and "plugins" in line:
            _lines = filter(None, re.split("\||\\s*\|\\s*\*", line))
            for line in _lines[1:]:
                label, value = _split(line, ":")
                if label == "Name":
                    plugin = copy.deepcopy(PLUGIN)
                    vuln_list.append(plugin)
                    plugin["name"] = value
                elif label == "Title":
                    plugin_vuln = copy.deepcopy(PLUGIN_OR_THEME_VULN)
                    plugin["vulnerabilities"] = plugin_vuln
                    plugin_vuln["title"] = value
                elif label == "Reference":
                    plugin_vuln["references"].append(value)
                elif label == "Fixed in":
                    plugin_vuln["fixed_since"] = value
            break
    return vuln_list

lines = re.split("\[\+\]|\[\!\]", stdout)
import pprint

pprint.pprint(get_wp_vuln(lines))
print "version: ", get_version(lines)
print "readme: ", is_readme_exists(lines)
print "theme in use: ", get_wp_theme_in_use(lines)
print "plugins: ", pprint.pprint(get_plugins(lines))
<|endoftext|>