<commit_before># Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved.
#
# Author: Kiall Mac Innes <kiall@hp.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from moniker.openstack.common import cfg

cfg.CONF.register_group(cfg.OptGroup(
    name='service:central', title="Configuration for Central Service"
))

cfg.CONF.register_opts([
    cfg.IntOpt('workers', default=None,
               help='Number of worker processes to spawn'),
    cfg.StrOpt('backend-driver', default='fake',
               help='The backend driver to use'),
    cfg.StrOpt('storage-driver', default='sqlalchemy',
               help='The storage driver to use'),
    cfg.ListOpt('enabled-notification-handlers', default=[],
                help='Enabled Notification Handlers'),
    cfg.ListOpt('domain-name-blacklist',
                default=['arpa.$', '^com.$', '^net.$', '^org.$',
                         'novalocal.$'],
                help='DNS domain name blacklist'),
    cfg.ListOpt('accepted-tld-list', default=None,
                help='Accepted TLDs'),
], group='service:central')
<commit_msg>Set the default value for accepted TLD's to match the IANA TLD list.<commit_after># Copyright 2012 Hewlett-Packard Development Company, L.P. All Rights Reserved.
#
# Author: Kiall Mac Innes <kiall@hp.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from moniker.openstack.common import cfg

# NOTE(kiall): See http://data.iana.org/TLD/tlds-alpha-by-domain.txt
#              Version 2013031800.
IANA_TLDS = [
    'ac', 'ad', 'ae', 'aero', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq',
    'ar', 'arpa', 'as', 'asia', 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd',
    'be', 'bf', 'bg', 'bh', 'bi', 'biz', 'bj', 'bm', 'bn', 'bo', 'br', 'bs',
    'bt', 'bv', 'bw', 'by', 'bz', 'ca', 'cat', 'cc', 'cd', 'cf', 'cg', 'ch',
    'ci', 'ck', 'cl', 'cm', 'cn', 'co', 'com', 'coop', 'cr', 'cu', 'cv', 'cw',
    'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', 'ec', 'edu', 'ee',
    'eg', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 'fo', 'fr', 'ga',
    'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 'gn', 'gov', 'gp',
    'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 'hr', 'ht',
    'hu', 'id', 'ie', 'il', 'im', 'in', 'info', 'int', 'io', 'iq', 'ir', 'is',
    'it', 'je', 'jm', 'jo', 'jobs', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn',
    'kp', 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls',
    'lt', 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mil', 'mk',
    'ml', 'mm', 'mn', 'mo', 'mobi', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu',
    'museum', 'mv', 'mw', 'mx', 'my', 'mz', 'na', 'name', 'nc', 'ne', 'net',
    'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', 'nz', 'om', 'org', 'pa',
    'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'post', 'pr', 'pro', 'ps',
    'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', 'sc',
    'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 'sr',
    'st', 'su', 'sv', 'sx', 'sy', 'sz', 'tc', 'td', 'tel', 'tf', 'tg', 'th',
    'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'tp', 'tr', 'travel', 'tt', 'tv', 'tw',
    'tz', 'ua', 'ug', 'uk', 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi',
    'vn', 'vu', 'wf', 'ws', 'xn--0zwm56d', 'xn--11b5bs3a9aj6g', 'xn--3e0b707e',
    'xn--45brj9c', 'xn--80akhbyknj4f', 'xn--80ao21a', 'xn--90a3ac',
    'xn--9t4b11yi5a', 'xn--clchc0ea0b2g2a9gcd', 'xn--deba0ad', 'xn--fiqs8s',
    'xn--fiqz9s', 'xn--fpcrj9c3d', 'xn--fzc2c9e2c', 'xn--g6w251d',
    'xn--gecrj9c', 'xn--h2brj9c', 'xn--hgbk6aj7f53bba', 'xn--hlcj6aya9esc7a',
    'xn--j6w193g', 'xn--jxalpdlp', 'xn--kgbechtv', 'xn--kprw13d',
    'xn--kpry57d', 'xn--lgbbat1ad8j', 'xn--mgb9awbf', 'xn--mgbaam7a8h',
    'xn--mgbayh7gpa', 'xn--mgbbh1a71e', 'xn--mgbc0a9azcg', 'xn--mgberp4a5d4ar',
    'xn--mgbx4cd0ab', 'xn--o3cw4h', 'xn--ogbpf8fl', 'xn--p1ai', 'xn--pgbs0dh',
    'xn--s9brj9c', 'xn--wgbh1c', 'xn--wgbl6a', 'xn--xkc2al3hye2a',
    'xn--xkc2dl3a5ee0h', 'xn--yfro4i67o', 'xn--ygbi2ammx', 'xn--zckzah', 'xxx',
    'ye', 'yt', 'za', 'zm', 'zw'
]

cfg.CONF.register_group(cfg.OptGroup(
    name='service:central', title="Configuration for Central Service"
))

cfg.CONF.register_opts([
    cfg.IntOpt('workers', default=None,
               help='Number of worker processes to spawn'),
    cfg.StrOpt('backend-driver', default='fake',
               help='The backend driver to use'),
    cfg.StrOpt('storage-driver', default='sqlalchemy',
               help='The storage driver to use'),
    cfg.ListOpt('enabled-notification-handlers', default=[],
                help='Enabled Notification Handlers'),
    cfg.ListOpt('domain-name-blacklist',
                default=['arpa.$', '^com.$', '^net.$', '^org.$',
                         'novalocal.$'],
                help='DNS domain name blacklist'),
    cfg.ListOpt('accepted-tld-list', default=IANA_TLDS,
                help='Accepted TLDs'),
], group='service:central')
<|endoftext|>


<commit_before>"""Fake mixer for use in tests.

**Dependencies**

None

**Configuration**

Set the :confval:`audio/mixe:` config value to ``fakemixer`` to use this mixer.
"""

from __future__ import unicode_literals

import pygst
pygst.require('0.10')
import gobject
import gst

from . import utils

class FakeMixer(gst.Element, gst.ImplementsInterface, gst.interfaces.Mixer):
    __gstdetails__ = (
        'FakeMixer',
        'Mixer',
        'Fake mixer for use in tests.',
        'Mopidy')

    track_label = gobject.property(type=str, default='Master')
    track_initial_volume = gobject.property(type=int, default=0)
    track_min_volume = gobject.property(type=int, default=0)
    track_max_volume = gobject.property(type=int, default=100)
    track_num_channels = gobject.property(type=int, default=2)
    track_flags = gobject.property(type=int, default=(
        gst.interfaces.MIXER_TRACK_MASTER | gst.interfaces.MIXER_TRACK_OUTPUT))

    def list_tracks(self):
        track = utils.create_track(
            self.track_label,
            self.track_initial_volume,
            self.track_min_volume,
            self.track_max_volume,
            self.track_num_channels,
            self.track_flags)
        return [track]

    def get_volume(self, track):
        return track.volumes

    def set_volume(self, track, volumes):
        track.volumes = volumes

    def set_record(self, track, record):
        pass
<commit_msg>docs: Fix typo<commit_after>"""Fake mixer for use in tests.

**Dependencies**

None

**Configuration**

Set the :confval:`audio/mixer:` config value to ``fakemixer`` to use this
mixer.
"""

from __future__ import unicode_literals

import pygst
pygst.require('0.10')
import gobject
import gst

from . import utils

class FakeMixer(gst.Element, gst.ImplementsInterface, gst.interfaces.Mixer):
    __gstdetails__ = (
        'FakeMixer',
        'Mixer',
        'Fake mixer for use in tests.',
        'Mopidy')

    track_label = gobject.property(type=str, default='Master')
    track_initial_volume = gobject.property(type=int, default=0)
    track_min_volume = gobject.property(type=int, default=0)
    track_max_volume = gobject.property(type=int, default=100)
    track_num_channels = gobject.property(type=int, default=2)
    track_flags = gobject.property(type=int, default=(
        gst.interfaces.MIXER_TRACK_MASTER | gst.interfaces.MIXER_TRACK_OUTPUT))

    def list_tracks(self):
        track = utils.create_track(
            self.track_label,
            self.track_initial_volume,
            self.track_min_volume,
            self.track_max_volume,
            self.track_num_channels,
            self.track_flags)
        return [track]

    def get_volume(self, track):
        return track.volumes

    def set_volume(self, track, volumes):
        track.volumes = volumes

    def set_record(self, track, record):
        pass
<|endoftext|>


<commit_before>from webob.exc import HTTPBadRequest
import webob.dec

FORWARDED_TOKENS = set(['by', 'for', 'host', 'proto'])

def handler_factory(handler):
    """Factory for handlers that set Host using RFC 7239 Forwarded header.

    :param handler: the handler that you want to wrap with Forwarded
      support
    :returns: the original ``handler`` wrapped with forwarded support.

    You can use this handler factory if you have an application that
    sits behind a trusted proxy that you know sets the RFC 7239
    Forwarded header. The Forwarded handler adjusts the Host header
    according to the information in this header.

    This makes the properties and methods on request that use this
    host information use the information the Forwarded header.

    This affects the properties and methods on :class:`BaseRequest`
    that use host information, such as :attr:`BaseRequest.host`,
    :attr:`BaseRequest.host_port`, :attr:`BaseRequest.host_url`,
    :attr:`BaseRequest.application_url`, :attr:`BaseRequest.path_url`,
    :attr:`BaseRequest.url`, :meth:`BaseRequest.relative_url`, and
    :attr:`BaseRequest.domain`.

    **Warning**: Use this handler factory *only* when you know your
    application lives behind a trusted proxy that sets the Forwarded
    header. The Forwarded header can be used by attackers to affect
    the URLs generated by your application and make them point to
    another host altogether.

    """
    def wrapped_handler(request):
        forwarded_header = request.headers.get('forwarded')
        if forwarded_header is None:
            return handler(request)
        try:
            forwarded = parse(forwarded_header)
        except ForwardedError:
            return HTTPBadRequest('Illegal Forwarded header')
        if not forwarded:
            return handler(request)
        item = forwarded[-1]
        forwarded_host = item.get('host')
        if forwarded_host is not None:
            request.host = forwarded_host
        forwarded_proto = item.get('proto')
        if forwarded_proto is not None:
            request.scheme = forwarded_proto
        return handler(request)
    return wrapped_handler

class Middleware(object):
    def __init__(self, app):
        def handler(request):
            return request.get_response(app)
        self.wrapped_handler = handler_factory(handler)

    @webob.dec.wsgify
    def __call__(self, request):
        return self.wrapped_handler(request)

class ForwardedError(ValueError):
    pass

def parse(header):
    """Parse the Forwarded header according to RFC 7239.

    Results in a list of dictionaries with the values in the header.
    """
    result = []
    for element in header.split(','):
        item = {}
        element = element.strip()
        if not element:
          continue
        for pair in element.split(';'):
            token, equals, value = pair.partition('=')
            if equals != '=':
                raise ForwardedError("Not a pair: %r" % pair)
            token = token.lower()
            # not a valid token, also deals with whitespace
            if token not in FORWARDED_TOKENS:
                raise ForwardedError("Unknown token: %r" % token)
            if token in item:
                raise ForwardedError("Duplicate token: %r" % token)
            if len(value.strip()) != len(value):
                raise ForwardedError(
                    "Value may not start or end with whitespace: %r" % value)
            # deal with quoted strings
            if value.startswith('"') and value.endswith('"'):
                value = value[1:-1]
            if value.startswith('"') or value.endswith('"'):
                raise ForwardedError(
                    "Value has unbalancd quotes: %r" % value)
            item[token] = value
        result.append(item)
    return result

<commit_msg>Get rid of PEP8 warnings<commit_after>from webob.exc import HTTPBadRequest
import webob.dec

FORWARDED_TOKENS = set(['by', 'for', 'host', 'proto'])

def handler_factory(handler):
    """Factory for handlers that set Host using RFC 7239 Forwarded header.

    :param handler: the handler that you want to wrap with Forwarded
      support
    :returns: the original ``handler`` wrapped with forwarded support.

    You can use this handler factory if you have an application that
    sits behind a trusted proxy that you know sets the RFC 7239
    Forwarded header. The Forwarded handler adjusts the Host header
    according to the information in this header.

    This makes the properties and methods on request that use this
    host information use the information the Forwarded header.

    This affects the properties and methods on :class:`BaseRequest`
    that use host information, such as :attr:`BaseRequest.host`,
    :attr:`BaseRequest.host_port`, :attr:`BaseRequest.host_url`,
    :attr:`BaseRequest.application_url`, :attr:`BaseRequest.path_url`,
    :attr:`BaseRequest.url`, :meth:`BaseRequest.relative_url`, and
    :attr:`BaseRequest.domain`.

    **Warning**: Use this handler factory *only* when you know your
    application lives behind a trusted proxy that sets the Forwarded
    header. The Forwarded header can be used by attackers to affect
    the URLs generated by your application and make them point to
    another host altogether.

    """
    def wrapped_handler(request):
        forwarded_header = request.headers.get('forwarded')
        if forwarded_header is None:
            return handler(request)
        try:
            forwarded = parse(forwarded_header)
        except ForwardedError:
            return HTTPBadRequest('Illegal Forwarded header')
        if not forwarded:
            return handler(request)
        item = forwarded[-1]
        forwarded_host = item.get('host')
        if forwarded_host is not None:
            request.host = forwarded_host
        forwarded_proto = item.get('proto')
        if forwarded_proto is not None:
            request.scheme = forwarded_proto
        return handler(request)
    return wrapped_handler

class Middleware(object):
    def __init__(self, app):
        def handler(request):
            return request.get_response(app)
        self.wrapped_handler = handler_factory(handler)

    @webob.dec.wsgify
    def __call__(self, request):
        return self.wrapped_handler(request)

class ForwardedError(ValueError):
    pass

def parse(header):
    """Parse the Forwarded header according to RFC 7239.

    Results in a list of dictionaries with the values in the header.
    """
    result = []
    for element in header.split(','):
        item = {}
        element = element.strip()
        if not element:
            continue
        for pair in element.split(';'):
            token, equals, value = pair.partition('=')
            if equals != '=':
                raise ForwardedError("Not a pair: %r" % pair)
            token = token.lower()
            # not a valid token, also deals with whitespace
            if token not in FORWARDED_TOKENS:
                raise ForwardedError("Unknown token: %r" % token)
            if token in item:
                raise ForwardedError("Duplicate token: %r" % token)
            if len(value.strip()) != len(value):
                raise ForwardedError(
                    "Value may not start or end with whitespace: %r" % value)
            # deal with quoted strings
            if value.startswith('"') and value.endswith('"'):
                value = value[1:-1]
            if value.startswith('"') or value.endswith('"'):
                raise ForwardedError(
                    "Value has unbalancd quotes: %r" % value)
            item[token] = value
        result.append(item)
    return result
<|endoftext|>


<commit_before>import os
import re

from .classes import PyModule
from .version import PY3
from .ast_scanner import ast_scan_file
from .frameworks import django
from . import utils, log

if not PY3:
    str = unicode

max_directory_depth = 20
max_file_size = 1024 * 1204

# Common ignorable directories
_dir_ignore = re.compile(r'(\.(git|hg|svn|tox)|CVS|__pycache__)\b')

# Files to not even bother with scanning
_ext_ignore = re.compile(r'\.(pyc|html|js|css|zip|tar(\.gz)?|txt|swp|~|bak|db)$', re.I)

def _scan_file(filename, sentinel, source_type='import'):
    '''Generator that performs the actual scanning of files.

    Yeilds a tuple containing import type, import path, and an extra file
    that should be scanned. Extra file scans should be the file or directory
    that relates to the import name.
    '''
    filename = os.path.abspath(filename)
    real_filename = os.path.realpath(filename)

    if os.path.getsize(filename) <= max_file_size:
        if real_filename not in sentinel and os.path.isfile(filename):
            sentinel.add(real_filename)

            basename = os.path.basename(filename)
            scope, imports = ast_scan_file(filename)

            if scope is not None and imports is not None:
                for imp in imports:
                    yield (source_type, imp.module, None)

                if 'INSTALLED_APPS' in scope and basename == 'settings.py':
                    log.info('Found Django settings: %s', filename)
                    for item in django.handle_django_settings(filename):
                        yield item
            else:
                log.warn('Could not scan imports from: %s', filename)
    else:
        log.warn('File size too large: %s', filename)

def _scan_directory(directory, sentinel, depth=0):
    '''Basically os.listdir with some filtering.
    '''
    directory = os.path.abspath(directory)
    real_directory = os.path.realpath(directory)

    if depth < max_directory_depth and real_directory not in sentinel \
            and os.path.isdir(directory):
        sentinel.add(real_directory)

        for item in os.listdir(directory):
            if item in ('.', '..'):
                # I'm not sure if this is even needed any more.
                continue

            p = os.path.abspath(os.path.join(directory, item))
            if (os.path.isdir(p) and _dir_ignore.search(p)) \
                    or (os.path.isfile(p) and _ext_ignore.search(p)):
                continue

            yield p

def scan_file(pym, filename, sentinel, installed):
    '''Entry point scan that creates a PyModule instance if needed.
    '''
    if not utils.is_python_script(filename):
        return

    if not pym:
        # This is for finding a previously created instance, not finding an
        # installed module with the same name. Might need to base the name
        # on the actual paths to reduce ambiguity in the printed scan results.
        module = os.path.basename(filename)
        pym = utils.find_package(module, installed)
        if not pym:
            pym = PyModule(module, 'SCRIPT', os.path.abspath(filename))
            installed.insert(0, pym)

    for imp_type, import_path, extra_file_scan in _scan_file(filename, sentinel):
        dep = utils.find_package(import_path, installed)
        if dep:
            dep.add_dependant(pym)
            pym.add_dependency(dep)

            if imp_type != 'import':
                pym.add_framework(imp_type)

        if extra_file_scan:
            # extra_file_scan should be a directory or file containing the
            # import name
            scan_filename = utils.file_containing_import(import_path, extra_file_scan)
            log.info('Related scan: %s - %s', import_path, scan_filename)
            if scan_filename.endswith('__init__.py'):
                scan_directory(pym, os.path.dirname(scan_filename), sentinel, installed)
            else:
                scan_file(pym, scan_filename, sentinel, installed)

    return pym

def scan_directory(pym, directory, sentinel, installed, depth=0):
    '''Entry point scan that creates a PyModule instance if needed.
    '''
    if not pym:
        d = os.path.abspath(directory)
        basename = os.path.basename(d)
        pym = utils.find_package(basename, installed)
        if not pym:
            version = 'DIRECTORY'
            if os.path.isfile(os.path.join(d, '__init__.py')):
                version = 'MODULE'
            pym = PyModule(basename, version, d)
            installed.insert(0, pym)

    # Keep track of how many file scans resulted in nothing
    bad_scans = 0

    for item in _scan_directory(directory, sentinel, depth):
        if os.path.isfile(item):
            if bad_scans > 100:
                # Keep in mind this counter resets if it a good scan happens
                # in *this* directory. If you have a module with more than 100
                # files in a single directory, you should probably refactor it.
                log.debug('Stopping scan of directory since it looks like a data dump: %s', directory)
                break

            if not scan_file(pym, item, sentinel, installed):
                bad_scans += 1
            else:
                bad_scans = 0
        elif os.path.isdir(item):
            scan_directory(pym, item, sentinel, installed, depth + 1)

    return pym

def scan(filename, installed, sentinel=None):
    if not sentinel:
        sentinel = set()

    if os.path.isfile(filename):
        return scan_file(None, filename, sentinel, installed)
    elif os.path.isdir(filename):
        return scan_directory(None, filename, sentinel, installed)
    else:
        log.error('Could not scan: %s', filename)
<commit_msg>If a package is found to be installed during a scan, mark it as scanned.<commit_after>import os
import re

from .classes import PyModule
from .version import PY3
from .ast_scanner import ast_scan_file
from .frameworks import django
from . import utils, log

if not PY3:
    str = unicode

max_directory_depth = 20
max_file_size = 1024 * 1204

# Common ignorable directories
_dir_ignore = re.compile(r'(\.(git|hg|svn|tox)|CVS|__pycache__)\b')

# Files to not even bother with scanning
_ext_ignore = re.compile(r'\.(pyc|html|js|css|zip|tar(\.gz)?|txt|swp|~|bak|db)$', re.I)

def _scan_file(filename, sentinel, source_type='import'):
    '''Generator that performs the actual scanning of files.

    Yeilds a tuple containing import type, import path, and an extra file
    that should be scanned. Extra file scans should be the file or directory
    that relates to the import name.
    '''
    filename = os.path.abspath(filename)
    real_filename = os.path.realpath(filename)

    if os.path.getsize(filename) <= max_file_size:
        if real_filename not in sentinel and os.path.isfile(filename):
            sentinel.add(real_filename)

            basename = os.path.basename(filename)
            scope, imports = ast_scan_file(filename)

            if scope is not None and imports is not None:
                for imp in imports:
                    yield (source_type, imp.module, None)

                if 'INSTALLED_APPS' in scope and basename == 'settings.py':
                    log.info('Found Django settings: %s', filename)
                    for item in django.handle_django_settings(filename):
                        yield item
            else:
                log.warn('Could not scan imports from: %s', filename)
    else:
        log.warn('File size too large: %s', filename)

def _scan_directory(directory, sentinel, depth=0):
    '''Basically os.listdir with some filtering.
    '''
    directory = os.path.abspath(directory)
    real_directory = os.path.realpath(directory)

    if depth < max_directory_depth and real_directory not in sentinel \
            and os.path.isdir(directory):
        sentinel.add(real_directory)

        for item in os.listdir(directory):
            if item in ('.', '..'):
                # I'm not sure if this is even needed any more.
                continue

            p = os.path.abspath(os.path.join(directory, item))
            if (os.path.isdir(p) and _dir_ignore.search(p)) \
                    or (os.path.isfile(p) and _ext_ignore.search(p)):
                continue

            yield p

def scan_file(pym, filename, sentinel, installed):
    '''Entry point scan that creates a PyModule instance if needed.
    '''
    if not utils.is_python_script(filename):
        return

    if not pym:
        # This is for finding a previously created instance, not finding an
        # installed module with the same name. Might need to base the name
        # on the actual paths to reduce ambiguity in the printed scan results.
        module = os.path.basename(filename)
        pym = utils.find_package(module, installed)
        if not pym:
            pym = PyModule(module, 'SCRIPT', os.path.abspath(filename))
            installed.insert(0, pym)
        else:
            pym.is_scan = True

    for imp_type, import_path, extra_file_scan in _scan_file(filename, sentinel):
        dep = utils.find_package(import_path, installed)
        if dep:
            dep.add_dependant(pym)
            pym.add_dependency(dep)

            if imp_type != 'import':
                pym.add_framework(imp_type)

        if extra_file_scan:
            # extra_file_scan should be a directory or file containing the
            # import name
            scan_filename = utils.file_containing_import(import_path, extra_file_scan)
            log.info('Related scan: %s - %s', import_path, scan_filename)
            if scan_filename.endswith('__init__.py'):
                scan_directory(pym, os.path.dirname(scan_filename), sentinel, installed)
            else:
                scan_file(pym, scan_filename, sentinel, installed)

    return pym

def scan_directory(pym, directory, sentinel, installed, depth=0):
    '''Entry point scan that creates a PyModule instance if needed.
    '''
    if not pym:
        d = os.path.abspath(directory)
        basename = os.path.basename(d)
        pym = utils.find_package(basename, installed)
        if not pym:
            version = 'DIRECTORY'
            if os.path.isfile(os.path.join(d, '__init__.py')):
                version = 'MODULE'
            pym = PyModule(basename, version, d)
            installed.insert(0, pym)
        else:
            pym.is_scan = True

    # Keep track of how many file scans resulted in nothing
    bad_scans = 0

    for item in _scan_directory(directory, sentinel, depth):
        if os.path.isfile(item):
            if bad_scans > 100:
                # Keep in mind this counter resets if it a good scan happens
                # in *this* directory. If you have a module with more than 100
                # files in a single directory, you should probably refactor it.
                log.debug('Stopping scan of directory since it looks like a data dump: %s', directory)
                break

            if not scan_file(pym, item, sentinel, installed):
                bad_scans += 1
            else:
                bad_scans = 0
        elif os.path.isdir(item):
            scan_directory(pym, item, sentinel, installed, depth + 1)

    return pym

def scan(filename, installed, sentinel=None):
    if not sentinel:
        sentinel = set()

    if os.path.isfile(filename):
        return scan_file(None, filename, sentinel, installed)
    elif os.path.isdir(filename):
        return scan_directory(None, filename, sentinel, installed)
    else:
        log.error('Could not scan: %s', filename)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
run talos tests in a virtualenv
"""

import os
import re

from mozharness.base.errors import PythonErrorList
from mozharness.base.log import DEBUG, ERROR, CRITICAL
from mozharness.base.python import virtualenv_config_options, VirtualenvMixin
from mozharness.base.script import BaseScript

TalosErrorList = PythonErrorList + [
 {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
 {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
 {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
 {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
 {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
 {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
 {'substr': r"""No such file or directory: 'browser_output.txt'""",
  'level': CRITICAL,
  'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
]

# TODO: check for running processes on script invocation

class Talos(VirtualenvMixin, BaseScript):
    """
    install and run Talos tests:
    https://wiki.mozilla.org/Buildbot/Talos
    """

    talos_options = [
        [["-a", "--tests"],
         {'action': 'extend',
          "dest": "tests",
          "default": [],
          "help": "Specify the tests to run"
          }],
        [["-e", "--binary"],
         {"action": "store",
          "dest": "binary",
          "default": None,
          "help": "Path to the binary to run tests on",
          }],
        [["--results-url"],
         {'action': 'store',
          'dest': 'results_url',
          'default': None,
          'help': "URL to send results to"
          }],
        ]

    config_options = [
        [["--talos-url"],
         {"action": "store",
          "dest": "talos_url",
          "default": "http://hg.mozilla.org/build/talos/archive/tip.tar.gz",
          "help": "Specify the talos package url"
          }],
        [["--add-option"],
          {"action": "extend",
           "dest": "talos_options",
           "default": None,
           "help": "extra options to PerfConfigurator"
           }],
        ] + talos_options + virtualenv_config_options

    actions = ['clobber',
               'create-virtualenv',
               'generate-config',
               'run-tests'
               ]

    def __init__(self, require_config_file=False):
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=self.actions,
                            default_actions=self.actions,
                            require_config_file=require_config_file,
                            config={"virtualenv_modules": ["talos"]},
                            )
        self.check() # basic setup and sanity check

        # results output
        self.results_url = self.config.get('results_url')
        if self.results_url is None:
            # use a results_url by default based on the class name in the working directory
            self.results_url = 'file://%s' % os.path.join(self.workdir, self.__class__.__name__.lower() + '.txt')

    def check(self):
        """ setup and sanity check"""

        self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience

        # path to browser
        self.binary = self.config.get('binary')
        if not self.binary:
            self.fatal("No path to binary specified; please specify --binary")
        self.binary = os.path.abspath(self.binary)
        if not os.path.exists(self.binary):
            self.fatal("Path to binary does not exist: %s" % self.binary)

        # Talos tests to run
        self.tests = self.config['tests']
        if not self.tests:
            self.fatal("No tests specified; please specify --tests")

    def PerfConfigurator_options(self, args=None, **kw):
        """return options to PerfConfigurator"""

        # TODO: do something about short options

        options = ['-v', '--develop'] # hardcoded options (for now)
        kw_options = {'output': 'talos.yml', # options overwritten from **kw
                      'executablePath': self.binary,
                      'activeTests': self.tests,
                      'results_url': self.results_url}
        if self.config.get('title'):
            kw_options['title'] = self.config['title']
        kw_options.update(kw)

        # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
        tests = kw_options['activeTests']
        if not isinstance(tests, basestring):
            tests = ':'.join(tests) # Talos expects this format
            kw_options['activeTests'] = tests

        for key, value in kw_options.items():
            options.extend(['--%s' % key, value])

        # extra arguments
        if args is None:
            args = self.config.get('perfconfigurator_options', [])
        options += args

        return options

    def talos_conf_path(self, conf):
        """return the full path for a talos .yml configuration file"""
        if os.path.isabs(conf):
            return conf
        return os.path.join(self.workdir, conf)

    def generate_config(self, conf='talos.yml', options=None):
        """generate talos configuration"""

        # XXX note: conf *must* match what is in options, if the latter is given

        # find the path to the talos .yml configuration
        # and remove if it exists
        talos_conf_path = self.talos_conf_path(conf)
        if os.path.exists(talos_conf_path):
            os.remove(talos_conf_path)

        # find PerfConfigurator console script
        # TODO: support remotePerfConfigurator if
        # https://bugzilla.mozilla.org/show_bug.cgi?id=704654
        # is not fixed first
        PerfConfigurator = self.query_python_path('PerfConfigurator')
        if not os.path.exists(PerfConfigurator):
            self.fatal("PerfConfigurator not found")

        # get command line for PerfConfigurator
        if options is None:
            options = self.PerfConfigurator_options(output=talos_conf_path)
        command = [PerfConfigurator] + options

        # run PerfConfigurator and ensure conf creation
        self.run_command(command, cwd=self.workdir,
                         error_list=TalosErrorList)
        if not os.path.exists(talos_conf_path):
            self.fatal("PerfConfigurator invokation failed: configuration file '%s' not found" % talos_conf_path)

    def run_tests(self, conf='talos.yml'):

        # generate configuration if necessary
        talos_conf_path = self.talos_conf_path(conf)
        if not os.path.exists(talos_conf_path):
            self.generate_config(conf)

        # run talos tests
        talos = self.query_python_path('talos')
        if not os.path.exists(talos):
            self.fatal("talos script not found")
        command = [talos, '--noisy', talos_conf_path]
        self.return_code = self.run_command(command, cwd=self.workdir,
                                            error_list=TalosErrorList)

if __name__ == '__main__':
    talos = Talos()
    talos.run()
<commit_msg>bug 742607 - fix unit.sh. r=jhammel<commit_after>#!/usr/bin/env python

"""
run talos tests in a virtualenv
"""

import os
import re

from mozharness.base.errors import PythonErrorList
from mozharness.base.log import DEBUG, ERROR, CRITICAL
from mozharness.base.python import virtualenv_config_options, VirtualenvMixin
from mozharness.base.script import BaseScript

TalosErrorList = PythonErrorList + [
 {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
 {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
 {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
 {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
 {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
 {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
 {'substr': r"""No such file or directory: 'browser_output.txt'""",
  'level': CRITICAL,
  'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
]

# TODO: check for running processes on script invocation

class Talos(VirtualenvMixin, BaseScript):
    """
    install and run Talos tests:
    https://wiki.mozilla.org/Buildbot/Talos
    """

    talos_options = [
        [["-a", "--tests"],
         {'action': 'extend',
          "dest": "tests",
          "default": [],
          "help": "Specify the tests to run"
          }],
        [["-e", "--binary"],
         {"action": "store",
          "dest": "binary",
          "default": None,
          "help": "Path to the binary to run tests on",
          }],
        [["--results-url"],
         {'action': 'store',
          'dest': 'results_url',
          'default': None,
          'help': "URL to send results to"
          }],
        ]

    config_options = [
        [["--talos-url"],
         {"action": "store",
          "dest": "talos_url",
          "default": "http://hg.mozilla.org/build/talos/archive/tip.tar.gz",
          "help": "Specify the talos package url"
          }],
        [["--add-option"],
          {"action": "extend",
           "dest": "talos_options",
           "default": None,
           "help": "extra options to PerfConfigurator"
           }],
        ] + talos_options + virtualenv_config_options

    actions = ['clobber',
               'create-virtualenv',
               'generate-config',
               'run-tests'
               ]

    def __init__(self, require_config_file=False):
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=self.actions,
                            default_actions=self.actions,
                            require_config_file=require_config_file,
                            config={"virtualenv_modules": ["talos"]},
                            )
        self.check() # basic setup and sanity check

        # results output
        self.results_url = self.config.get('results_url')
        if self.results_url is None:
            # use a results_url by default based on the class name in the working directory
            self.results_url = 'file://%s' % os.path.join(self.workdir, self.__class__.__name__.lower() + '.txt')

    def check(self):
        """ setup and sanity check"""

        self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience

        # path to browser
        self.binary = self.config.get('binary')
        if not self.binary:
            self.fatal("No path to binary specified; please specify --binary")
        self.binary = os.path.abspath(self.binary)
        if not os.path.exists(self.binary):
            self.fatal("Path to binary does not exist: %s" % self.binary)

        # Talos tests to run
        self.tests = self.config['tests']
        if not self.tests:
            self.fatal("No tests specified; please specify --tests")

    def PerfConfigurator_options(self, args=None, **kw):
        """return options to PerfConfigurator"""

        # TODO: do something about short options

        options = ['-v', '--develop'] # hardcoded options (for now)
        kw_options = {'output': 'talos.yml', # options overwritten from **kw
                      'executablePath': self.binary,
                      'activeTests': self.tests,
                      'results_url': self.results_url}
        if self.config.get('title'):
            kw_options['title'] = self.config['title']
        kw_options.update(kw)

        # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
        tests = kw_options['activeTests']
        if not isinstance(tests, basestring):
            tests = ':'.join(tests) # Talos expects this format
            kw_options['activeTests'] = tests

        for key, value in kw_options.items():
            options.extend(['--%s' % key, value])

        # extra arguments
        if args is None:
            args = self.config.get('perfconfigurator_options', [])
        options += args

        return options

    def talos_conf_path(self, conf):
        """return the full path for a talos .yml configuration file"""
        if os.path.isabs(conf):
            return conf
        return os.path.join(self.workdir, conf)

    def generate_config(self, conf='talos.yml', options=None):
        """generate talos configuration"""

        # XXX note: conf *must* match what is in options, if the latter is given

        # find the path to the talos .yml configuration
        # and remove if it exists
        talos_conf_path = self.talos_conf_path(conf)
        if os.path.exists(talos_conf_path):
            os.remove(talos_conf_path)

        # find PerfConfigurator console script
        # TODO: support remotePerfConfigurator if
        # https://bugzilla.mozilla.org/show_bug.cgi?id=704654
        # is not fixed first
        PerfConfigurator = self.query_python_path('PerfConfigurator')
        if not os.path.exists(PerfConfigurator):
            self.fatal("PerfConfigurator not found")

        # get command line for PerfConfigurator
        if options is None:
            options = self.PerfConfigurator_options(output=talos_conf_path)
        command = [PerfConfigurator] + options

        # run PerfConfigurator and ensure conf creation
        self.run_command(command, cwd=self.workdir,
                         error_list=TalosErrorList)
        if not os.path.exists(talos_conf_path):
            self.fatal("PerfConfigurator invokation failed: configuration file '%s' not found" % talos_conf_path)

    def run_tests(self, conf='talos.yml'):

        # generate configuration if necessary
        talos_conf_path = self.talos_conf_path(conf)
        if not os.path.exists(talos_conf_path):
            self.generate_config(conf)

        # run talos tests
        talos = self.query_python_path('talos')
        if not os.path.exists(talos):
            self.fatal("talos script not found")
        command = [talos, '--noisy', talos_conf_path]
        self.return_code = self.run_command(command, cwd=self.workdir,
                                            error_list=TalosErrorList)
<|endoftext|>


<commit_before><commit_msg>making sure bumper monitor does not try to restart motors after finding a barrier<commit_after><|endoftext|>


<commit_before>"""Contains the Blinkenlight class."""
import asyncio

from typing import Set, Dict, List, Tuple, Any, NoReturn
from operator import itemgetter

from mpf.core.rgb_color import RGBColor
from mpf.core.system_wide_device import SystemWideDevice
from mpf.core.delays import DelayManager
from mpf.core.device_monitor import DeviceMonitor
from mpf.exceptions.config_file_error import ConfigFileError

@DeviceMonitor("num_colors", "light")
class Blinkenlight(SystemWideDevice):

    """A light that alternates between several different colors."""

    config_section = 'blinkenlights'
    collection = 'blinkenlights'
    class_label = 'blinkenlight'

    def __init__(self, machine, name):
        """Initialise blinkenlight."""
        super().__init__(machine, name)

        self._colors = []
        self.delay = DelayManager(machine)
        self._num_colors = 0
        self._color_duration = None
        self._cycle_duration = None

    def load_config(self, config: dict):
        """Load config."""
        super().load_config(config)

        self._color_duration = self.config['color_duration']
        self._cycle_duration = self.config['cycle_duration']
        if (self._color_duration is None and self._cycle_duration is None) or \
           (self._color_duration is not None and self._cycle_duration is not None):
            self._blinkenlight_validation_error(
                "Either color_duration or cycle_duration must be specified, but not both.", 1)

    def _blinkenlight_validation_error(self, msg, error_code) -> "NoReturn":  # pragma: no cover
        raise ConfigFileError('"{}" >> {}'.format(self.name, msg), error_code, "blinkenlight", self.name)

    @property
    def num_colors(self):
        """Return the number of colors (excluding "off") for this blinkenlight."""
        return self._num_colors

    @num_colors.setter
    def num_colors(self, newvalue):
        self._num_colors = newvalue
        # if the number of colors is 0, remove us from the light stack
        if self._num_colors == 0:
            self.light.remove_from_stack_by_key(self._blinkenlight_key)

    @property
    def light(self):
        """Return the light this blinkenlight controls."""
        return self.config['light']

    @property
    def num_colors_in_cycle(self):
        """Similar to num_colors, but adds 1 for the "off" color if there is one between cycles."""
        return self.num_colors + (1 if self._off_between_cycles() else 0)

    @property
    def _blinkenlight_key(self):
        # this is the key that's passed to the light as the color's key
        # so that we can remove this blinkenlight's color from the light stack
        # when all the blinkenlight's colors are removed.
        # This prevents the light from staying in the off state if the blinkenlight
        # was on top of the priority list.
        return 'blinkenlight_{}'.format(self.name)

    def add_color(self, color, key, priority):
        """Add a color to the blinkenlight."""
        # check if this key already exists. If it does, replace it with the incoming color/priority
        existing_color_with_key = [x for x in self._colors if x[1] == key]
        if len(existing_color_with_key) == 0:
            self._colors.append((color, key, priority))
            self.num_colors += 1
            self.info_log('Color {} with key {} added'.format(color, key))
            self._restart()
        elif len(existing_color_with_key) == 1:
            # color with this key already exists. Just update it with this new color and priority
            self._remove_color_with_key(key)
            self.add_color(color, key, priority)

    def remove_all_colors(self):
        """Remove all colors from the blinkenlight."""
        self._colors.clear()
        self.num_colors = 0
        self.info_log('All colors removed')
        self._restart()

    def remove_color_with_key(self, key):
        """Remove a color with a given key from the blinkenlight."""
        color = [x for x in self._colors if x[1] == key]
        if len(color) == 1:
            self._colors.remove(color[0])
            self.num_colors -= 1
            self.info_log('Color removed with key {}'.format(key))
            self._restart()

    def _restart(self):
        self.delay.clear()
        self._sort_colors()
        self._perform_step()

    def _sort_colors(self):
        """Sort the blinkenlights colors by their priority."""
        self._colors = sorted(self._colors, key=itemgetter(2), reverse=True)  # priority is item 2 of the tuple

    def _off_between_cycles(self):
        return (self.config['off_when_multiple'] and len(self._colors) > 1) or (len(self._colors) == 1)

    def _get_time_between_colors_ms(self):
        if self._cycle_duration:
            delay = self._cycle_duration / self.num_colors_in_cycle
        else:
            delay = self._color_duration
        if delay < 1:
            delay = 1
        return delay

    def _get_total_cycle_ms(self):
        if self._cycle_duration:
            return self._cycle_duration
        return self._color_duration * self.num_colors_in_cycle

    def _get_current_color(self):
        now = self.machine.clock.get_time()
        offset_ms = (now * 1000) % self._get_total_cycle_ms()
        color_i = int(offset_ms / self._get_time_between_colors_ms() + 0.5)
        if color_i >= self.num_colors:
            return RGBColor("off")
        return self._colors[color_i][0]

    def _perform_step(self):
        if len(self._colors) == 0:
            return
        light = self.config['light']
        current_color = self._get_current_color()
        color_duration_ms = self._get_time_between_colors_ms()
        cycle_ms = self._get_total_cycle_ms()
        if current_color is None:
            return
        light.color(current_color, priority=self.config['priority'], key=self._blinkenlight_key)
        delay_ms = color_duration_ms - ((self.machine.clock.get_time() * 1000) % cycle_ms) % color_duration_ms
        self.delay.add(delay_ms, self._perform_step)
<commit_msg>Removed unused import.<commit_after>"""Contains the Blinkenlight class."""
import asyncio

from typing import Set, Dict, List, Tuple, Any
from operator import itemgetter

from mpf.core.rgb_color import RGBColor
from mpf.core.system_wide_device import SystemWideDevice
from mpf.core.delays import DelayManager
from mpf.core.device_monitor import DeviceMonitor
from mpf.exceptions.config_file_error import ConfigFileError

@DeviceMonitor("num_colors", "light")
class Blinkenlight(SystemWideDevice):

    """A light that alternates between several different colors."""

    config_section = 'blinkenlights'
    collection = 'blinkenlights'
    class_label = 'blinkenlight'

    def __init__(self, machine, name):
        """Initialise blinkenlight."""
        super().__init__(machine, name)

        self._colors = []
        self.delay = DelayManager(machine)
        self._num_colors = 0
        self._color_duration = None
        self._cycle_duration = None

    def load_config(self, config: dict):
        """Load config."""
        super().load_config(config)

        self._color_duration = self.config['color_duration']
        self._cycle_duration = self.config['cycle_duration']
        if (self._color_duration is None and self._cycle_duration is None) or \
           (self._color_duration is not None and self._cycle_duration is not None):
            self._blinkenlight_validation_error(
                "Either color_duration or cycle_duration must be specified, but not both.", 1)

    def _blinkenlight_validation_error(self, msg, error_code) -> "NoReturn":  # pragma: no cover
        raise ConfigFileError('"{}" >> {}'.format(self.name, msg), error_code, "blinkenlight", self.name)

    @property
    def num_colors(self):
        """Return the number of colors (excluding "off") for this blinkenlight."""
        return self._num_colors

    @num_colors.setter
    def num_colors(self, newvalue):
        self._num_colors = newvalue
        # if the number of colors is 0, remove us from the light stack
        if self._num_colors == 0:
            self.light.remove_from_stack_by_key(self._blinkenlight_key)

    @property
    def light(self):
        """Return the light this blinkenlight controls."""
        return self.config['light']

    @property
    def num_colors_in_cycle(self):
        """Similar to num_colors, but adds 1 for the "off" color if there is one between cycles."""
        return self.num_colors + (1 if self._off_between_cycles() else 0)

    @property
    def _blinkenlight_key(self):
        # this is the key that's passed to the light as the color's key
        # so that we can remove this blinkenlight's color from the light stack
        # when all the blinkenlight's colors are removed.
        # This prevents the light from staying in the off state if the blinkenlight
        # was on top of the priority list.
        return 'blinkenlight_{}'.format(self.name)

    def add_color(self, color, key, priority):
        """Add a color to the blinkenlight."""
        # check if this key already exists. If it does, replace it with the incoming color/priority
        existing_color_with_key = [x for x in self._colors if x[1] == key]
        if len(existing_color_with_key) == 0:
            self._colors.append((color, key, priority))
            self.num_colors += 1
            self.info_log('Color {} with key {} added'.format(color, key))
            self._restart()
        elif len(existing_color_with_key) == 1:
            # color with this key already exists. Just update it with this new color and priority
            self._remove_color_with_key(key)
            self.add_color(color, key, priority)

    def remove_all_colors(self):
        """Remove all colors from the blinkenlight."""
        self._colors.clear()
        self.num_colors = 0
        self.info_log('All colors removed')
        self._restart()

    def remove_color_with_key(self, key):
        """Remove a color with a given key from the blinkenlight."""
        color = [x for x in self._colors if x[1] == key]
        if len(color) == 1:
            self._colors.remove(color[0])
            self.num_colors -= 1
            self.info_log('Color removed with key {}'.format(key))
            self._restart()

    def _restart(self):
        self.delay.clear()
        self._sort_colors()
        self._perform_step()

    def _sort_colors(self):
        """Sort the blinkenlights colors by their priority."""
        self._colors = sorted(self._colors, key=itemgetter(2), reverse=True)  # priority is item 2 of the tuple

    def _off_between_cycles(self):
        return (self.config['off_when_multiple'] and len(self._colors) > 1) or (len(self._colors) == 1)

    def _get_time_between_colors_ms(self):
        if self._cycle_duration:
            delay = self._cycle_duration / self.num_colors_in_cycle
        else:
            delay = self._color_duration
        if delay < 1:
            delay = 1
        return delay

    def _get_total_cycle_ms(self):
        if self._cycle_duration:
            return self._cycle_duration
        return self._color_duration * self.num_colors_in_cycle

    def _get_current_color(self):
        now = self.machine.clock.get_time()
        offset_ms = (now * 1000) % self._get_total_cycle_ms()
        color_i = int(offset_ms / self._get_time_between_colors_ms() + 0.5)
        if color_i >= self.num_colors:
            return RGBColor("off")
        return self._colors[color_i][0]

    def _perform_step(self):
        if len(self._colors) == 0:
            return
        light = self.config['light']
        current_color = self._get_current_color()
        color_duration_ms = self._get_time_between_colors_ms()
        cycle_ms = self._get_total_cycle_ms()
        if current_color is None:
            return
        light.color(current_color, priority=self.config['priority'], key=self._blinkenlight_key)
        delay_ms = color_duration_ms - ((self.machine.clock.get_time() * 1000) % cycle_ms) % color_duration_ms
        self.delay.add(delay_ms, self._perform_step)
<|endoftext|>


<commit_before>"""
Example generation from python files.

Generate the rst files for the examples by iterating over the python
example files. Files that generate images should start with 'plot'.

To generate your own examples, just add ``'mpltools.sphinx.plot2rst'``` to the
list of ``extensions``in your Sphinx configuration file.

This code was adapted from scikits-image, which took it from scikits-learn.

Options
-------
The ``plot2rst`` extension accepts the following options:

    plot2rst_paths : length-2 tuple
        Paths to (python plot, generated rst) files, i.e. (source, destination).
        Note that both paths are relative to Sphinx 'source' path.

    plot2rst_rcparams : dict
        Matplotlib configuration parameters. See
        http://matplotlib.sourceforge.net/users/customizing.html for details.
"""
import os
import shutil
import traceback
import glob

import matplotlib
matplotlib.use('Agg')

plot_rst_template = """
.. _example_%(short_filename)s:

%(docstring)s

%(image_list)s

**Python source code:** :download:`%(src_name)s <%(src_name)s>`
(generated using ``mpltools`` |version|)

.. literalinclude:: %(src_name)s
    :lines: %(end_row)s-

"""

toctree_template = """
.. toctree::
   :hidden:

   %s

"""

CLEAR_SECTION = """
.. raw:: html

    <div style="clear: both"></div>

"""

IMAGE_TEMPLATE = """
.. image:: images/%s
    :align: center

"""

GALLERY_HEADER = """

Examples
========

.. _examples-index:

"""

def setup(app):
    app.connect('builder-inited', generate_rst_gallery)

    app.add_config_value('plot2rst_paths',
                         ('../examples', 'auto_examples'), True)
    app.add_config_value('plot2rst_rcparams', {}, True)
    app.add_config_value('plot2rst_default_thumb', None, True)

def generate_rst_gallery(app):
    """Add list of examples and gallery to Sphinx app."""
    cfg = app.builder.config

    doc_src = os.path.abspath(app.builder.srcdir) # path/to/doc/source

    plot_path, rst_path = cfg.plot2rst_paths
    rst_dir = os.path.join(doc_src, rst_path)
    example_dir = os.path.join(doc_src, plot_path)

    if not os.path.exists(example_dir):
        print "No example directory found at", example_dir
        return
    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    # we create an index.rst with all examples
    gallery_index = file(os.path.join(rst_dir, 'index'+cfg.source_suffix), 'w')
    gallery_index.write(GALLERY_HEADER)

    # Here we don't use an os.walk, but we recurse only twice: flat is
    # better than nested.
    write_gallery(gallery_index, example_dir, rst_dir, cfg)
    for d in sorted(os.listdir(example_dir)):
        example_sub = os.path.join(example_dir, d)
        if os.path.isdir(example_sub):
            rst_sub = os.path.join(rst_dir, d)
            if not os.path.exists(rst_sub):
                os.makedirs(rst_sub)
            write_gallery(gallery_index, example_sub, rst_sub, cfg, depth=1)
    gallery_index.flush()

def write_gallery(gallery_index, src_dir, rst_dir, cfg, depth=0):
    """Generate the rst files for an example directory, i.e. gallery.

    Write rst files from python examples and add example links to gallery.

    Parameters
    ----------
    gallery_index : file
        Index file for plot gallery.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    index_name = 'index' + cfg.source_suffix
    if not os.path.exists(os.path.join(src_dir, index_name)):
        print src_dir
        print 80*'_'
        print ('Example directory %s does not have a %s file'
                        % (src_dir, index_name))
        print 'Skipping this directory'
        print 80*'_'
        return

    gallery_description = file(os.path.join(src_dir, index_name)).read()
    gallery_index.write('\n\n%s\n\n' % gallery_description)

    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    def sort_key(a):
        # Elements that are not plots should be last.
        if not valid_plot_script(a):
            return 'zz' + a
        return a

    examples = [fname for fname in sorted(os.listdir(src_dir), key=sort_key)
                      if fname.endswith('py')]
    ex_names = [ex[:-3] for ex in examples] # strip '.py' extension
    if depth == 0:
        sub_dir = ''
    else:
        sub_dir_list = src_dir.split('/')[-depth:]
        sub_dir = '/'.join(sub_dir_list) + '/'
    gallery_index.write(toctree_template % (sub_dir + '\n   '.join(ex_names)))

    write = gallery_index.write
    for src_name in examples:
        rst_file_from_example(src_name, src_dir, rst_dir, cfg)
        thumb = os.path.join(sub_dir, 'images/thumb', src_name[:-3] + '.png')
        gallery_index.write('.. figure:: %s\n' % thumb)

        link_name = sub_dir + src_name
        link_name = link_name.replace(os.path.sep, '_')
        if link_name.startswith('._'):
            link_name = link_name[2:]

        write('   :figclass: gallery\n')
        write('   :target: ./%s.html\n\n' % (sub_dir + src_name[:-3]))
        write('   :ref:`example_%s`\n\n' % (link_name))
    write(CLEAR_SECTION) # clear at the end of the section

def valid_plot_script(src_name):
    return src_name.startswith('plot') and src_name.endswith('.py')

def rst_file_from_example(src_name, src_dir, rst_dir, cfg):
    """Write rst file from a given python example.

    Parameters
    ----------
    src_name : str
        Name of example file.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    base_image_name = os.path.splitext(src_name)[0]
    image_fmt_str = '%s_%%s.png' % base_image_name

    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'

    info = dict(src_name=src_name)
    info['short_filename'] = last_dir + src_name
    src_path = os.path.join(src_dir, src_name)
    example_file = os.path.join(rst_dir, src_name)
    shutil.copyfile(src_path, example_file)

    image_dir = os.path.join(rst_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_path = os.path.join(thumb_dir, src_name[:-3] + '.png')
    image_path = os.path.join(image_dir, image_fmt_str)
    figure_list = save_plot(src_path, image_path, thumb_path, cfg)

    if not os.path.exists(thumb_path):
        if cfg.plot2rst_default_thumb is None:
            print "WARNING: No plots found and default thumbnail not defined."
            print "Specify 'plot2rst_default_thumb' in Sphinx config file."
        else:
            shutil.copy(cfg.plot2rst_default_thumb, thumb_path)

    blocks = split_code_and_text(example_file)
    first_text_block = [b for b in blocks if b[0] == 'text'][0]
    label, (start, end), content = first_text_block
    info['docstring'] = content.strip().strip('"""')
    info['end_row'] = end + 1

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.

    image_rst_directives = []
    for figure_name in figure_list:
        image_rst_directives.append(IMAGE_TEMPLATE % figure_name.lstrip('/'))
    info['image_list'] = ''.join(image_rst_directives)

    basename, py_ext = os.path.splitext(src_name)
    f = open(os.path.join(rst_dir, basename + cfg.source_suffix),'w')
    f.write(plot_rst_template % info)
    f.flush()

def split_code_and_text(source_file):
    """Return list with source file separated into code and text blocks.

    Returns
    -------
    blocks : list of (label, (start, end+1), content)
        List where each element is a tuple with the label ('text' or 'code'),
        the (start, end+1) line numbers, and content string of block.
    """
    with open(source_file) as f:
        source_lines = f.readlines()
    blocks = []
    i = 0
    last_line = len(source_lines)
    while True:
        if start_of_text(source_lines[i]):
            label = 'text'
            token = source_lines[i][:3] + '\n' # either """\n or '''\n
            j = _end_index(i, lambda k: source_lines[k].endswith(token))
            j += 1 # set j to line after text block
        else:
            label = 'code'
            j = _end_index(i, lambda k: start_of_text(source_lines[k]))
        # Add 1 to convert list indices to line numbers, which start at 1.
        blocks.append((label, (i+1, j+1), ''.join(source_lines[i:j])))
        i = j
        if i == last_line:
            break
    return blocks

def _end_index(i, stop_condition):
    j = i
    while True:
        j += 1
        try:
            if stop_condition(j):
                return j
        except IndexError:
            return j

def start_of_text(line):
    return line.startswith('"""') or line.startswith("'''")

def save_plot(src_path, image_path, thumb_path, cfg):
    """Save plots as images.

    Parameters
    ----------
    src_path : str
        Path to example file.
    image_path : str
        Path where plots are saved (format string with single string argument).
    thumbpath : str
        Path where thumbnails of plots are saved.
    cfg : config object
        Sphinx config object created by Sphinx.

    Returns
    -------
    figure_list : list
        List of figure names saved by the example.
    """
    figure_list = []

    src_dir, src_name = os.path.split(src_path)
    if not src_name.startswith('plot'):
        return figure_list

    image_dir, image_fmt_str = os.path.split(image_path)
    first_image_file = image_path % 1

    needs_replot = (not os.path.exists(first_image_file) or
                    mod_time(first_image_file) <= mod_time(src_path))
    if needs_replot:
        print 'plotting %s' % src_name
        import matplotlib.pyplot as plt
        plt.rcdefaults()
        plt.rcParams.update(cfg.plot2rst_rcparams)
        plt.close('all')
        cwd = os.getcwd()
        try:
            # Plot example in source directory.
            os.chdir(src_dir)
            execfile(src_name, {'pl' : plt})
            os.chdir(cwd)

            fig_mngr = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
            # Save every figure by looping over all open figures.
            for fig_num in (m.num for m in fig_mngr):
                # Set the fig_num figure as the current figure as we can't
                # save a figure that's not the current figure.
                plt.figure(fig_num)
                plt.savefig(image_path % fig_num)
                figure_list.append(image_fmt_str % fig_num)
        except:
            print 80*'_'
            print '%s is not compiling:' % src_name
            traceback.print_exc()
            print 80*'_'
        finally:
            os.chdir(cwd)
    else:
        figure_list = [f[len(image_dir):]
                        for f in glob.glob(image_path % '[1-9]')]

    # generate thumb file
    from matplotlib import image
    if os.path.exists(first_image_file):
        image.thumbnail(first_image_file, thumb_path, 0.2)

    return figure_list

def mod_time(file_path):
    return os.stat(file_path).st_mtime

<commit_msg>Move code block in preparation for segmenting plots.<commit_after>"""
Example generation from python files.

Generate the rst files for the examples by iterating over the python
example files. Files that generate images should start with 'plot'.

To generate your own examples, just add ``'mpltools.sphinx.plot2rst'``` to the
list of ``extensions``in your Sphinx configuration file.

This code was adapted from scikits-image, which took it from scikits-learn.

Options
-------
The ``plot2rst`` extension accepts the following options:

    plot2rst_paths : length-2 tuple
        Paths to (python plot, generated rst) files, i.e. (source, destination).
        Note that both paths are relative to Sphinx 'source' path.

    plot2rst_rcparams : dict
        Matplotlib configuration parameters. See
        http://matplotlib.sourceforge.net/users/customizing.html for details.
"""
import os
import shutil
import traceback
import glob

import matplotlib
matplotlib.use('Agg')

plot_rst_template = """
.. _example_%(short_filename)s:

%(docstring)s

%(image_list)s

**Python source code:** :download:`%(src_name)s <%(src_name)s>`
(generated using ``mpltools`` |version|)

.. literalinclude:: %(src_name)s
    :lines: %(end_row)s-

"""

toctree_template = """
.. toctree::
   :hidden:

   %s

"""

CLEAR_SECTION = """
.. raw:: html

"""

IMAGE_TEMPLATE = """
.. image:: images/%s
    :align: center

"""

GALLERY_HEADER = """

Examples
========

.. _examples-index:

"""

def setup(app):
    app.connect('builder-inited', generate_rst_gallery)

    app.add_config_value('plot2rst_paths',
                         ('../examples', 'auto_examples'), True)
    app.add_config_value('plot2rst_rcparams', {}, True)
    app.add_config_value('plot2rst_default_thumb', None, True)

def generate_rst_gallery(app):
    """Add list of examples and gallery to Sphinx app."""
    cfg = app.builder.config

    doc_src = os.path.abspath(app.builder.srcdir) # path/to/doc/source

    plot_path, rst_path = cfg.plot2rst_paths
    rst_dir = os.path.join(doc_src, rst_path)
    example_dir = os.path.join(doc_src, plot_path)

    if not os.path.exists(example_dir):
        print "No example directory found at", example_dir
        return
    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    # we create an index.rst with all examples
    gallery_index = file(os.path.join(rst_dir, 'index'+cfg.source_suffix), 'w')
    gallery_index.write(GALLERY_HEADER)

    # Here we don't use an os.walk, but we recurse only twice: flat is
    # better than nested.
    write_gallery(gallery_index, example_dir, rst_dir, cfg)
    for d in sorted(os.listdir(example_dir)):
        example_sub = os.path.join(example_dir, d)
        if os.path.isdir(example_sub):
            rst_sub = os.path.join(rst_dir, d)
            if not os.path.exists(rst_sub):
                os.makedirs(rst_sub)
            write_gallery(gallery_index, example_sub, rst_sub, cfg, depth=1)
    gallery_index.flush()

def write_gallery(gallery_index, src_dir, rst_dir, cfg, depth=0):
    """Generate the rst files for an example directory, i.e. gallery.

    Write rst files from python examples and add example links to gallery.

    Parameters
    ----------
    gallery_index : file
        Index file for plot gallery.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    index_name = 'index' + cfg.source_suffix
    if not os.path.exists(os.path.join(src_dir, index_name)):
        print src_dir
        print 80*'_'
        print ('Example directory %s does not have a %s file'
                        % (src_dir, index_name))
        print 'Skipping this directory'
        print 80*'_'
        return

    gallery_description = file(os.path.join(src_dir, index_name)).read()
    gallery_index.write('\n\n%s\n\n' % gallery_description)

    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    def sort_key(a):
        # Elements that are not plots should be last.
        if not valid_plot_script(a):
            return 'zz' + a
        return a

    examples = [fname for fname in sorted(os.listdir(src_dir), key=sort_key)
                      if fname.endswith('py')]
    ex_names = [ex[:-3] for ex in examples] # strip '.py' extension
    if depth == 0:
        sub_dir = ''
    else:
        sub_dir_list = src_dir.split('/')[-depth:]
        sub_dir = '/'.join(sub_dir_list) + '/'
    gallery_index.write(toctree_template % (sub_dir + '\n   '.join(ex_names)))

    write = gallery_index.write
    for src_name in examples:
        rst_file_from_example(src_name, src_dir, rst_dir, cfg)
        thumb = os.path.join(sub_dir, 'images/thumb', src_name[:-3] + '.png')
        gallery_index.write('.. figure:: %s\n' % thumb)

        link_name = sub_dir + src_name
        link_name = link_name.replace(os.path.sep, '_')
        if link_name.startswith('._'):
            link_name = link_name[2:]

        write('   :figclass: gallery\n')
        write('   :target: ./%s.html\n\n' % (sub_dir + src_name[:-3]))
        write('   :ref:`example_%s`\n\n' % (link_name))
    write(CLEAR_SECTION) # clear at the end of the section

def valid_plot_script(src_name):
    return src_name.startswith('plot') and src_name.endswith('.py')

def rst_file_from_example(src_name, src_dir, rst_dir, cfg):
    """Write rst file from a given python example.

    Parameters
    ----------
    src_name : str
        Name of example file.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    base_image_name = os.path.splitext(src_name)[0]
    image_fmt_str = '%s_%%s.png' % base_image_name

    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'

    info = dict(src_name=src_name)
    info['short_filename'] = last_dir + src_name
    src_path = os.path.join(src_dir, src_name)
    example_file = os.path.join(rst_dir, src_name)
    shutil.copyfile(src_path, example_file)

    blocks = split_code_and_text(example_file)
    first_text_block = [b for b in blocks if b[0] == 'text'][0]
    label, (start, end), content = first_text_block
    info['docstring'] = content.strip().strip('"""')
    info['end_row'] = end + 1

    image_dir = os.path.join(rst_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_path = os.path.join(thumb_dir, src_name[:-3] + '.png')
    image_path = os.path.join(image_dir, image_fmt_str)
    figure_list = save_plot(src_path, image_path, thumb_path, cfg)

    if not os.path.exists(thumb_path):
        if cfg.plot2rst_default_thumb is None:
            print "WARNING: No plots found and default thumbnail not defined."
            print "Specify 'plot2rst_default_thumb' in Sphinx config file."
        else:
            shutil.copy(cfg.plot2rst_default_thumb, thumb_path)

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.

    image_rst_directives = []
    for figure_name in figure_list:
        image_rst_directives.append(IMAGE_TEMPLATE % figure_name.lstrip('/'))
    info['image_list'] = ''.join(image_rst_directives)

    basename, py_ext = os.path.splitext(src_name)
    f = open(os.path.join(rst_dir, basename + cfg.source_suffix),'w')
    f.write(plot_rst_template % info)
    f.flush()

def split_code_and_text(source_file):
    """Return list with source file separated into code and text blocks.

    Returns
    -------
    blocks : list of (label, (start, end+1), content)
        List where each element is a tuple with the label ('text' or 'code'),
        the (start, end+1) line numbers, and content string of block.
    """
    with open(source_file) as f:
        source_lines = f.readlines()
    blocks = []
    i = 0
    last_line = len(source_lines)
    while True:
        if start_of_text(source_lines[i]):
            label = 'text'
            token = source_lines[i][:3] + '\n' # either """\n or '''\n
            j = _end_index(i, lambda k: source_lines[k].endswith(token))
            j += 1 # set j to line after text block
        else:
            label = 'code'
            j = _end_index(i, lambda k: start_of_text(source_lines[k]))
        # Add 1 to convert list indices to line numbers, which start at 1.
        blocks.append((label, (i+1, j+1), ''.join(source_lines[i:j])))
        i = j
        if i == last_line:
            break
    return blocks

def _end_index(i, stop_condition):
    j = i
    while True:
        j += 1
        try:
            if stop_condition(j):
                return j
        except IndexError:
            return j

def start_of_text(line):
    return line.startswith('"""') or line.startswith("'''")

def save_plot(src_path, image_path, thumb_path, cfg):
    """Save plots as images.

    Parameters
    ----------
    src_path : str
        Path to example file.
    image_path : str
        Path where plots are saved (format string with single string argument).
    thumbpath : str
        Path where thumbnails of plots are saved.
    cfg : config object
        Sphinx config object created by Sphinx.

    Returns
    -------
    figure_list : list
        List of figure names saved by the example.
    """
    figure_list = []

    src_dir, src_name = os.path.split(src_path)
    if not src_name.startswith('plot'):
        return figure_list

    image_dir, image_fmt_str = os.path.split(image_path)
    first_image_file = image_path % 1

    needs_replot = (not os.path.exists(first_image_file) or
                    mod_time(first_image_file) <= mod_time(src_path))
    if needs_replot:
        print 'plotting %s' % src_name
        import matplotlib.pyplot as plt
        plt.rcdefaults()
        plt.rcParams.update(cfg.plot2rst_rcparams)
        plt.close('all')
        cwd = os.getcwd()
        try:
            # Plot example in source directory.
            os.chdir(src_dir)
            execfile(src_name, {'pl' : plt})
            os.chdir(cwd)

            fig_mngr = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
            # Save every figure by looping over all open figures.
            for fig_num in (m.num for m in fig_mngr):
                # Set the fig_num figure as the current figure as we can't
                # save a figure that's not the current figure.
                plt.figure(fig_num)
                plt.savefig(image_path % fig_num)
                figure_list.append(image_fmt_str % fig_num)
        except:
            print 80*'_'
            print '%s is not compiling:' % src_name
            traceback.print_exc()
            print 80*'_'
        finally:
            os.chdir(cwd)
    else:
        figure_list = [f[len(image_dir):]
                        for f in glob.glob(image_path % '[1-9]')]

    # generate thumb file
    from matplotlib import image
    if os.path.exists(first_image_file):
        image.thumbnail(first_image_file, thumb_path, 0.2)

    return figure_list

def mod_time(file_path):
    return os.stat(file_path).st_mtime

<|endoftext|>


<commit_before>#!/usr/bin/python3

"""File format classes for the Commander Keen: Invasion of the Vorticons engine (DOS, 1991)"""

import itertools

from mrcrowbar import models as mrc
from mrcrowbar.lib.hardware import ibm_pc
from mrcrowbar.lib.images import base as img
from mrcrowbar import utils

# source: http://www.shikadi.net/moddingwiki/Keen_1-3_RLE_compression

class RLECompressor( mrc.Transform ):
    def import_data( self, buffer, parent=None ):
        final_length = utils.from_uint32_le( buffer[0:4] )
        i = 4
        out = bytearray()
        while (len( out ) < final_length):
            byte = buffer[i]
            if byte >= 128:
                out.extend( buffer[i+1:i+byte-126] )
                i += byte-126
            else:
                out.extend( buffer[i+1:i+2]*(byte+3) )
                i += 2

        result = {
            'payload': bytes( out ),
            'end_offset': i
        }

        return result

# source: http://www.shikadi.net/moddingwiki/RLEW_compression

class RLEWCompressor( mrc.Transform ):
    def import_data( self, buffer, parent=None ):
        final_length = utils.from_uint32_le( buffer[0:4] )
        i = 4
        out = bytearray()
        while (len( out ) < final_length):
            word = buffer[i:i+2]
            if word == b'\xfe\xfe':
                count = utils.from_uint16_le( buffer[i+2:i+4] )
                data = buffer[i+4:i+6]
                out.extend( data*count )
                i += 6
            else:
                out.extend( word )
                i += 2

        result = {
            'payload': bytes( out ),
            'end_offset': i
        }

        return result

# source: http://www.shikadi.net/moddingwiki/Commander_Keen_EGA_Header

class EGAHead( mrc.Block ):
    latch_plane_size    = mrc.UInt32_LE( 0x00 )
    sprite_plane_size   = mrc.UInt32_LE( 0x04 )
    image_data_start    = mrc.UInt32_LE( 0x08 )
    sprite_data_start   = mrc.UInt32_LE( 0x0c )
    font_count          = mrc.UInt16_LE( 0x10 )
    font_offset         = mrc.UInt32_LE( 0x12 )
    unknown_1           = mrc.UInt16_LE( 0x16 )
    unknown_2           = mrc.UInt32_LE( 0x18 )
    tile_count          = mrc.UInt16_LE( 0x1a )
    tile_offset         = mrc.UInt32_LE( 0x1c )
    bitmap_count        = mrc.UInt16_LE( 0x22 )
    bitmap_offset       = mrc.UInt32_LE( 0x24 )
    sprite_count        = mrc.UInt16_LE( 0x28 )
    sprite_offset       = mrc.UInt32_LE( 0x2a )
    latch_compressed    = mrc.Bits( 0x2e, 0b00000010 )
    sprite_compressed   = mrc.Bits( 0x2e, 0b00000001 )

class EGAHeadBitmapRef( mrc.Block ):
    width_raw           = mrc.UInt16_LE( 0x00 )
    height              = mrc.UInt16_LE( 0x02 )
    location            = mrc.UInt32_LE( 0x04 )
    name                = mrc.CStringN( 0x08, 8 )

class EGAHeadSpriteRef( mrc.Block ):
    width_raw           = mrc.UInt16_LE( 0x00 )
    height              = mrc.UInt16_LE( 0x02 )
    prog_offset         = mrc.UInt16_LE( 0x04 )
    location_raw        = mrc.UInt16_LE( 0x06 )
    hitbox_left         = mrc.UInt16_LE( 0x08 )
    hitbox_top          = mrc.UInt16_LE( 0x0a )
    hitbox_right        = mrc.UInt16_LE( 0x0c )
    httbox_bottom       = mrc.UInt16_LE( 0x0e )
    name                = mrc.CStringN( 0x10, 12 )
    horiz_offset        = mrc.UInt16_LE( 0x1c )
    vert_offset         = mrc.UInt16_LE( 0x1e )

    @property
    def location( self ):
        return self.location_raw*16

class PreviewCompressor( mrc.Transform ):
    rle = RLECompressor()
    # each plane is stored with 192 bytes padding at the end
    plan = img.Planarizer( 320, 200, 4, plane_padding=192 )

    def import_data( self, buffer, parent=None ):
        assert utils.is_bytes( buffer )
        stage_1 = self.rle.import_data( buffer )
        stage_2 = self.plan.import_data( stage_1['payload'] )
        result = {
            'payload': stage_2['payload'],
            'end_offset': stage_1['end_offset']
        }
        return result

class Preview( mrc.Block ):
    image_data          = mrc.Bytes( 0x0000, transform=PreviewCompressor() )

    def __init__( self, *args, **kwargs ):
        mrc.Block.__init__( self, *args, **kwargs )
        self.image = img.IndexedImage( self, width=320, height=200, palette=ibm_pc.EGA_DEFAULT_PALETTE, source=mrc.Ref( 'image_data' ) )

# source: http://www.shikadi.net/moddingwiki/Commander_Keen_1-3_Level_format

class Level( mrc.Block ):
    pass

class LevelHeader( mrc.Block ):
    width           = mrc.UInt16_LE( 0x00 )
    height          = mrc.UInt16_LE( 0x02 )
    plane_count     = mrc.Check( mrc.UInt16_LE( 0x04 ), 2 )
    plane_size      = mrc.UInt16_LE( 0x0e )

class LevelTile( mrc.Block ):
    tile_id         = mrc.UInt16_LE( 0x00 )

class Loader( mrc.Loader ):
    _SEP = mrc.Loader._SEP

    _KEEN_FILE_CLASS_MAP = {
        _SEP+'(EGAHEAD).CK[1-3]$': EGAHead,
        _SEP+'(FINALE).CK[1-3]$': Preview,
        _SEP+'(PREVIEW)([2-3]).CK1': Preview
    }

    def __init__( self ):
        super( Loader, self ).__init__( self._KEEN_FILE_CLASS_MAP )
<commit_msg>lib.games.keen: stub out more formats<commit_after>#!/usr/bin/python3

"""File format classes for the Commander Keen: Invasion of the Vorticons engine (DOS, 1991)

Sources:
RLE compressor
http://www.shikadi.net/moddingwiki/Keen_1-3_RLE_compression

RLEW compressor
http://www.shikadi.net/moddingwiki/RLEW_compression

LZW compressor
http://www.shikadi.net/moddingwiki/LZW_Compression
(Special thanks to Fleexy)

EGA header
http://www.shikadi.net/moddingwiki/Commander_Keen_EGA_Header

Level format
http://www.shikadi.net/moddingwiki/Commander_Keen_1-3_Level_format
"""

import itertools

from mrcrowbar import models as mrc
from mrcrowbar.lib.hardware import ibm_pc
from mrcrowbar.lib.images import base as img
from mrcrowbar import utils

class RLECompressor( mrc.Transform ):
    def import_data( self, buffer, parent=None ):
        final_length = utils.from_uint32_le( buffer[0:4] )
        i = 4
        out = bytearray()
        while (len( out ) < final_length):
            byte = buffer[i]
            if byte >= 128:
                out.extend( buffer[i+1:i+byte-126] )
                i += byte-126
            else:
                out.extend( buffer[i+1:i+2]*(byte+3) )
                i += 2

        result = {
            'payload': bytes( out ),
            'end_offset': i
        }

        return result

class RLEWCompressor( mrc.Transform ):
    def import_data( self, buffer, parent=None ):
        final_length = utils.from_uint32_le( buffer[0:4] )
        i = 4
        out = bytearray()
        while (len( out ) < final_length):
            word = buffer[i:i+2]
            if word == b'\xfe\xfe':
                count = utils.from_uint16_le( buffer[i+2:i+4] )
                data = buffer[i+4:i+6]
                out.extend( data*count )
                i += 6
            else:
                out.extend( word )
                i += 2

        result = {
            'payload': bytes( out ),
            'end_offset': i
        }

        return result

class LZWCompressor( mrc.Transform ):

    def import_data( self, buffer, parent=None ):
        decomp_size = utils.from_uint32_le( buffer[:4] )
        max_bits = utils.from_uint16_le( buffer[4:6] )      # should be 12

        lookup = [bytes((i,)) for i in range( 256 )]
        lookup.append( None )   # 256: error
        lookup.append( None )   # 257: end of data

        output = bytearray()

        bs = utils.BitReader( buffer, 6, bits_reverse=True, output_reverse=True )
        state = {'usebits': 9}

        def add_to_lookup( state, entry ):
            if len( lookup ) < (1 << max_bits):
                #print('lookup[{}] = {}'.format( len( lookup ), entry ) )
                lookup.append( entry )
                if len( lookup ) == (1 << state['usebits'])-1:
                    state['usebits'] = min( state['usebits']+1, max_bits )
                    #print('usebits = {}'.format(state['usebits']))
            return

        fcode = bs.get_bits( state['usebits'] )
        match = lookup[fcode]
        #print( 'fcode={},match={}'.format( fcode, match ) )
        output.extend( match )
        while True:
            ncode = bs.get_bits( state['usebits'] )
            #print( 'ncode={}'.format( ncode ) )
            if ncode == 257:
                # end of data
                break
            elif ncode == 256:
                # error
                raise Exception( 'Found error code, data is not valid' )
            elif ncode < len( lookup ):
                nmatch = lookup[ncode]
            else:
                nmatch = match+match[0:1]
            #print( 'match={}'.format(match) )
            #print( 'nmatch={}'.format(nmatch) )
            output.extend( nmatch )

            # add code to lookup
            add_to_lookup( state, match+nmatch[0:1] )
            match = nmatch

        if len( output ) != decomp_size:
            print( 'Warning: was expecting data of size {}, got data of size {} instead'.format( decomp_size, len( output ) ) )

        result = {
            'payload': bytes( output ),
            'end_offset': bs.pos
        }

        return result

class EGAHeader( mrc.Block ):
    latch_plane_size    = mrc.UInt32_LE( 0x00 )
    sprite_plane_size   = mrc.UInt32_LE( 0x04 )
    image_data_start    = mrc.UInt32_LE( 0x08 )
    sprite_data_start   = mrc.UInt32_LE( 0x0c )
    tile8_count         = mrc.UInt16_LE( 0x10 )
    tile8_offset        = mrc.UInt32_LE( 0x12 )
    tile32_count        = mrc.UInt16_LE( 0x16 )
    tile32_offset       = mrc.UInt32_LE( 0x18 )
    tile16_count        = mrc.UInt16_LE( 0x1c )
    tile16_offset       = mrc.UInt32_LE( 0x1e )
    bitmap_count        = mrc.UInt16_LE( 0x22 )
    bitmap_offset       = mrc.UInt32_LE( 0x24 )
    sprite_count        = mrc.UInt16_LE( 0x28 )
    sprite_offset       = mrc.UInt32_LE( 0x2a )
    latch_compressed    = mrc.Bits( 0x2e, 0b00000010 )
    sprite_compressed   = mrc.Bits( 0x2e, 0b00000001 )

class EGAHeaderBitmapRef( mrc.Block ):
    width_raw           = mrc.UInt16_LE( 0x00 )
    height              = mrc.UInt16_LE( 0x02 )
    location            = mrc.UInt32_LE( 0x04 )
    name                = mrc.Bytes( 0x08, length=8 )

class EGAHeaderSpriteRef( mrc.Block ):
    width_raw           = mrc.UInt16_LE( 0x00 )
    height              = mrc.UInt16_LE( 0x02 )
    prog_offset         = mrc.UInt16_LE( 0x04 )
    location_raw        = mrc.UInt16_LE( 0x06 )
    hitbox_left         = mrc.UInt16_LE( 0x08 )
    hitbox_top          = mrc.UInt16_LE( 0x0a )
    hitbox_right        = mrc.UInt16_LE( 0x0c )
    httbox_bottom       = mrc.UInt16_LE( 0x0e )
    name                = mrc.Bytes( 0x10, length=12 )
    horiz_offset        = mrc.UInt16_LE( 0x1c )
    vert_offset         = mrc.UInt16_LE( 0x1e )

    @property
    def location( self ):
        return self.location_raw*16

class EGATiles( mrc.Block ):
    pass
#    tile_data           =   mrc.Bytes(
#                                0x00, transform=img.Planarizer(
#                                    width=32,
#                                    height,
#                                    bpp=4,
#                                    frame_count
#                                )
#                            )

class EGALatch( mrc.Block ):
    tiles               = mrc.BlockField( EGATiles, 0x00 )

class EGALatchComp( mrc.Block ):
    tiles               = mrc.BlockField( EGATiles, 0x00, transform=LZWCompressor() )

class SoundHeader( mrc.Block ):
    magic               = mrc.Const( mrc.Bytes( 0x00, length=4 ), b'SND\x00' )
    size                = mrc.UInt16_LE( 0x04 )
    unknown_1           = mrc.UInt16_LE( 0x06 )
    count               = mrc.UInt16_LE( 0x08 )
    padding             = mrc.Const( mrc.Bytes( 0x0a, length=6 ), b'\x00'*6 )

class SoundRef( mrc.Block ):
    offset              = mrc.UInt16_LE( 0x00 )
    priority            = mrc.UInt8( 0x02 )
    rate                = mrc.UInt8( 0x03 )
    name                = mrc.Bytes( 0x04, 0x0c )

class PreviewCompressor( mrc.Transform ):
    rle = RLECompressor()
    # each plane is stored with 192 bytes padding at the end
    plan = img.Planarizer( 320, 200, 4, plane_padding=192 )

    def import_data( self, buffer, parent=None ):
        assert utils.is_bytes( buffer )
        stage_1 = self.rle.import_data( buffer )
        stage_2 = self.plan.import_data( stage_1['payload'] )
        result = {
            'payload': stage_2['payload'],
            'end_offset': stage_1['end_offset']
        }
        return result

class Preview( mrc.Block ):
    image_data          = mrc.Bytes( 0x0000, transform=PreviewCompressor() )

    def __init__( self, *args, **kwargs ):
        mrc.Block.__init__( self, *args, **kwargs )
        self.image = img.IndexedImage( self, width=320, height=200, palette=ibm_pc.EGA_DEFAULT_PALETTE, source=mrc.Ref( 'image_data' ) )

# source: http://www.shikadi.net/moddingwiki/Commander_Keen_1-3_Level_format

class Level( mrc.Block ):
    pass

class LevelHeader( mrc.Block ):
    width           = mrc.UInt16_LE( 0x00 )
    height          = mrc.UInt16_LE( 0x02 )
    plane_count     = mrc.UInt16_LE( 0x04, default=2 )
    unknown_1       = mrc.Bytes( 0x06, length=4 )
    ted_vars        = mrc.Bytes( 0x0a, length=4 )
    plane_size      = mrc.UInt16_LE( 0x0e )
    unknown_2       = mrc.Bytes( 0x10, length=16 )

    @property
    def plane_size_calc( self ):
        size = 2 * self.width * self.height
        return size + ((16-size)%16)

class LevelTile( mrc.Block ):
    tile_id         = mrc.UInt16_LE( 0x00 )

class ScoresItems( mrc.Block ):
    joystick        = mrc.UInt16_LE( 0x00, range( 0, 1 ) )
    battery         = mrc.UInt16_LE( 0x02, range( 0, 1 ) )
    vacuum          = mrc.UInt16_LE( 0x04, range( 0, 1 ) )
    liquor          = mrc.UInt16_LE( 0x06, range( 0, 1 ) )

class ScoresName( mrc.Block ):
    name            = mrc.Bytes( 0x00, length=13 )

class Scores( mrc.Block ):
    values          = mrc.UInt32_LE( 0x00, length=7 )
    items           = mrc.BlockField( ScoresItems, 0x1c, count=7 )
    num_cities      = mrc.UInt16_LE( 0x54, length=7, range=range( 0, 9 ) )
    unknown_1       = mrc.Bytes( 0x62, length=14 )
    names           = mrc.BlockField( ScoresName, 0x70, count=7 )
    term            = mrc.Const( mrc.Bytes( 0xcb, length=1 ), b'\x00' )

class Loader( mrc.Loader ):
    _SEP = mrc.Loader._SEP

    _KEEN_FILE_CLASS_MAP = {
        _SEP+'(EGAHEAD).CK[1-3]$': EGAHeader,
        _SEP+'(EGALATCH).CK1$': EGALatchComp,
        _SEP+'(EGALATCH).CK[2-3]$': EGALatch,
        _SEP+'(FINALE).CK[1-3]$': Preview,
        _SEP+'(LEVEL)([0-9]{2}).CK[1-3]$': Level,
        _SEP+'(PREVIEW)([2-3]).CK[1-3]$': Preview,
        _SEP+'(SCORES).CK[1-3]$': Scores,
    }

    def __init__( self ):
        super( Loader, self ).__init__( self._KEEN_FILE_CLASS_MAP )
<|endoftext|>