<commit_before>#!/usr/bin/env python
# coding=utf-8

"""
ola channel mapper.

    read a configuration file and map channels from one universe to a second.

    history:
        see git commits

    todo:
        ~ all fine :-)
"""

import sys
import time
import os
import array
import json

from configdict import ConfigDict
from olathreaded import OLAThread, OLAThread_States

version = """08.03.2016 12:30 stefan"""

##########################################
# globals

##########################################
# functions

##########################################
# classes

class OLAMapper(OLAThread):
    """Class that extends on OLAThread and implements the Mapper functions."""

    def __init__(self, config):
        """init mapper things."""
        # super(OLAThread, self).__init__()
        OLAThread.__init__(self)

        self.config = config
        # print("config: {}".format(self.config))

        self.universe = self.config['universe']['output']
        # self.channel_count = 512
        # self.channel_count = 50
        self.channel_count = self.config['universe']['channel_count']
        self.channels_out = array.array('B')

        # internal map
        self.map = []
        self.map_create()
        # print("full map: {}".format(map_tostring_pretty()))

        # self.channels = []
        for channel_index in range(0, self.channel_count):
            self.channels_out.append(0)

        # timing things
        self.duration = 0
        self.calls = 0

    def print_measurements(self):
        """print duration statistics on exit."""
        # print duration meassurements:
        if self.calls > 0:
            print(
                (
                    "map_channels:\n" +
                    "  sum duration:  {:>10f}s\n" +
                    "  sum calls:     {:>10}\n" +
                    "  duration/call: {:>10.2f}ms/call\n"
                ).format(
                    self.duration,
                    self.calls,
                    ((self.duration / self.calls)*1000)
                )
            )

    def ola_connected(self):
        """register receive callback and switch to running mode."""
        self.client.RegisterUniverse(
            self.config['universe']['input'],
            self.client.REGISTER,
            self.dmx_receive_frame
        )
        # python3 syntax
        # super().ola_connected()
        # python2 syntax
        # super(OLAThread, self).ola_connected()
        # explicit call
        OLAThread.ola_connected(self)

    def dmx_receive_frame(self, data):
        """receive one dmx frame."""
        # print(data)
        # meassure duration:
        start = time.time()
        self.map_channels(data)
        stop = time.time()
        duration = stop - start
        self.duration += duration
        self.calls += 1
        # temp_array = array.array('B')
        # for channel_index in range(0, self.channel_count):
        #     temp_array.append(self.channels[channel_index])

    def map_create(self):
        """create map based on configuration."""
        # self.map
        map_config = self.config['map']

        data_output = array.array('B')

        # for channel_index in range(0, data_input_length):
        #     data_output.append(data_input[channel_index])
        channel_output_count_temp = len(map_config['channels'])
        if map_config['repeat'] is True:
            channel_output_count_temp = self.channel_count
        elif isinstance(map_config['repeat'], int):
            channel_output_count_temp = (
                len(map_config['channels']) * map_config['repeat']
            )

        for channel_output_index in range(0, channel_output_count_temp):
            # calculate map_index
            map_index = channel_output_index % len(map_config['channels'])
            # print("map_index: {}".format(map_index))

            # get map_config channel
            map_value = map_config['channels'][map_index]
            if map_value is not -1:
                if map_config['repeat'] and map_config['offset']:
                    loop_index = (
                        channel_output_index // len(map_config['channels'])
                    )
                    if (
                        isinstance(map_config['repeat'], int) and
                        map_config['repeat_reverse']
                    ):
                        map_value = map_value + (
                            ((map_config['repeat']-1) - loop_index) *
                            map_config['offset_count']
                        )
                    else:
                        map_value = (
                            map_value +
                            (loop_index * map_config['offset_count'])
                        )
            # print("map_value: {}".format(map_value))

            # add channel to map
            self.map.append(map_value)

    def map_tostring_pretty(self):
        """print map content in pretty way."""
        output = ""
        map_config = self.config['map']
        if isinstance(map_config['offset_count'], int):
            array = ""
            separator_line = "\n  "
            array += separator_line
            for index, value in enumerate(self.map):
                if (index is len(self.map)-1):
                    array += "{: >3}".format(value)
                else:
                    array += "{: >3}, ".format(value)
                # after every repeat break line
                if (
                    (((index+1) % map_config['offset_count']) == 0) and
                    not (index+1 is len(self.map))
                ):
                    # print("index: {}".format(index))
                    array += separator_line
            output = "[{}\n]".format(array)
        else:
            # print(output)
            output = json.dumps(
                self.map,
                sort_keys=True,
                indent=4,
                separators=(',', ': ')
            )
        # [ array, content ]

        # print("map: {}".format(output))
        return output

    def map_channels(self, data_input):
        """remap channels according to map tabel."""
        # print("map channels:")
        # print("data_input: {}".format(data_input))
        data_input_length = data_input.buffer_info()[1]
        # print("data_input_length: {}".format(data_input_length))
        # print("map: {}".format(self.config['map']))

        for channel_output_index, map_value in enumerate(self.map):

            # check if map_value is in range of input channels
            if (
                # (map_value < data_input_length) and
                (map_value < data_input_length)
            ):
                try:
                    self.channels_out[channel_output_index] = (
                        data_input[map_value]
                    )
                except Exception as e:
                    print(
                        (
                            "additional info:\n" +
                            "  channel_output_index: {}\n" +
                            "  len(self.channels_out): {}\n" +
                            "  map_value: {}\n"
                        ).format(
                            channel_output_index,
                            len(self.channels_out),
                            map_value
                        )
                    )
                    raise
            else:
                # don't alter data
                pass

        self.dmx_send_frame(
            self.config['universe']['output'],
            self.channels_out
        )

##########################################
if __name__ == '__main__':

    print(42*'*')
    print('Python Version: ' + sys.version)
    print(42*'*')
    print(__doc__)
    print(42*'*')

    # parse arguments
    filename = "map.json"
    # only use args after script name
    arg = sys.argv[1:]
    if not arg:
        print("using standard values.")
        print(" Allowed parameters:")
        print("   filename for config file       (default='map.json')")
        print("")
    else:
        filename = arg[0]
        # if len(arg) > 1:
        #     pixel_count = int(arg[1])
    # print parsed argument values
    print('''values:
        filename :{}
    '''.format(filename))

    default_config = {
        'universe': {
            'input': 1,
            'output': 2,
            'channel_count': 240,
        },
        'map': {
            'channels': [
                -1,
                -1,
                -1,
                -1,
                30,
                31,
                -1,
                -1,
                -1,
                -1,
                22,
                23,
                -1,
                -1,
                -1,
                -1,
                14,
                15,
                -1,
                -1,
                -1,
                -1,
                6,
                7,
                28,
                29,
                26,
                27,
                24,
                25,
                20,
                21,
                18,
                19,
                16,
                17,
                12,
                13,
                10,
                11,
                8,
                9,
                4,
                5,
                2,
                3,
                0,
                1,
            ],
            'repeat': 5,
            'repeat_reverse': True,
            'offset': True,
            'offset_count': 48,
        },
    }
    my_config = ConfigDict(default_config, filename)
    print("my_config.config: {}".format(my_config.config))

    my_mapper = OLAMapper(my_config.config)
    print("full map:\n{}".format(my_mapper.map_tostring_pretty()))

    my_mapper.start_ola()

    # wait for user to hit key.
    try:
        raw_input(
            "\n\n" +
            42*'*' +
            "\nhit a key to stop the mapper\n" +
            42*'*' +
            "\n\n"
        )
    except KeyboardInterrupt:
        print("\nstop.")
    except:
        print("\nstop.")

    # blocks untill thread has joined.
    my_mapper.stop_ola()

    my_mapper.print_measurements()

    # as last thing we save the current configuration.
    print("\nwrite config.")
    my_config.write_to_file()

    # ###########################################
<commit_msg>fix mapping<commit_after>#!/usr/bin/env python
# coding=utf-8

"""
ola channel mapper.

    read a configuration file and map channels from one universe to a second.

    history:
        see git commits

    todo:
        ~ all fine :-)
"""

import sys
import time
import os
import array
import json

from configdict import ConfigDict
from olathreaded import OLAThread, OLAThread_States

version = """08.03.2016 12:30 stefan"""

##########################################
# globals

##########################################
# functions

##########################################
# classes

class OLAMapper(OLAThread):
    """Class that extends on OLAThread and implements the Mapper functions."""

    def __init__(self, config):
        """init mapper things."""
        # super(OLAThread, self).__init__()
        OLAThread.__init__(self)

        self.config = config
        # print("config: {}".format(self.config))

        self.universe = self.config['universe']['output']
        # self.channel_count = 512
        # self.channel_count = 50
        self.channel_count = self.config['universe']['channel_count']
        self.channels_out = array.array('B')

        # internal map
        self.map = []
        self.map_create()
        # print("full map: {}".format(map_tostring_pretty()))

        # self.channels = []
        for channel_index in range(0, self.channel_count):
            self.channels_out.append(0)

        # timing things
        self.duration = 0
        self.calls = 0

    def print_measurements(self):
        """print duration statistics on exit."""
        # print duration meassurements:
        if self.calls > 0:
            print(
                (
                    "map_channels:\n" +
                    "  sum duration:  {:>10f}s\n" +
                    "  sum calls:     {:>10}\n" +
                    "  duration/call: {:>10.2f}ms/call\n"
                ).format(
                    self.duration,
                    self.calls,
                    ((self.duration / self.calls)*1000)
                )
            )

    def ola_connected(self):
        """register receive callback and switch to running mode."""
        self.client.RegisterUniverse(
            self.config['universe']['input'],
            self.client.REGISTER,
            self.dmx_receive_frame
        )
        # python3 syntax
        # super().ola_connected()
        # python2 syntax
        # super(OLAThread, self).ola_connected()
        # explicit call
        OLAThread.ola_connected(self)

    def dmx_receive_frame(self, data):
        """receive one dmx frame."""
        # print(data)
        # meassure duration:
        start = time.time()
        self.map_channels(data)
        stop = time.time()
        duration = stop - start
        self.duration += duration
        self.calls += 1
        # temp_array = array.array('B')
        # for channel_index in range(0, self.channel_count):
        #     temp_array.append(self.channels[channel_index])

    def map_create(self):
        """create map based on configuration."""
        # self.map
        map_config = self.config['map']

        data_output = array.array('B')

        # for channel_index in range(0, data_input_length):
        #     data_output.append(data_input[channel_index])
        channel_output_count_temp = len(map_config['channels'])
        if map_config['repeat'] is True:
            channel_output_count_temp = self.channel_count
        elif isinstance(map_config['repeat'], int):
            channel_output_count_temp = (
                len(map_config['channels']) * map_config['repeat']
            )

        for channel_output_index in range(0, channel_output_count_temp):
            # calculate map_index
            map_index = channel_output_index % len(map_config['channels'])
            # print("map_index: {}".format(map_index))

            # get map_config channel
            map_value = map_config['channels'][map_index]
            if map_value is not -1:
                if map_config['repeat'] and map_config['offset']:
                    loop_index = (
                        channel_output_index // len(map_config['channels'])
                    )
                    if (
                        isinstance(map_config['repeat'], int) and
                        map_config['repeat_reverse']
                    ):
                        map_value = map_value + (
                            ((map_config['repeat']-1) - loop_index) *
                            map_config['offset_count']
                        )
                    else:
                        map_value = (
                            map_value +
                            (loop_index * map_config['offset_count'])
                        )
            # print("map_value: {}".format(map_value))

            # add channel to map
            self.map.append(map_value)

    def map_tostring_pretty(self):
        """print map content in pretty way."""
        output = ""
        map_config = self.config['map']
        array = ""
        separator_line = "\n  "
        array += separator_line
        for index, value in enumerate(self.map):
            if (index is len(self.map)-1):
                array += "{: >3}".format(value)
            else:
                array += "{: >3}, ".format(value)
            # after every repeat break line
            if (
                (((index+1) % len(map_config['channels'])) == 0) and
                not (index+1 is len(self.map))
            ):
                # print("index: {}".format(index))
                array += separator_line
        output = "[{}\n]".format(array)
        # else:
        #     # print(output)
        #     output = json.dumps(
        #         self.map,
        #         sort_keys=True,
        #         indent=4,
        #         separators=(',', ': ')
        #     )
        # [ array, content ]
        # print("map: {}".format(output))
        return output

    def map_channels(self, data_input):
        """remap channels according to map tabel."""
        # print("map channels:")
        # print("data_input: {}".format(data_input))
        data_input_length = data_input.buffer_info()[1]
        # print("data_input_length: {}".format(data_input_length))
        # print("map: {}".format(self.config['map']))

        for channel_output_index, map_value in enumerate(self.map):

            # check if map_value is in range of input channels
            if (
                # (map_value < data_input_length) and
                (map_value < data_input_length)
            ):
                try:
                    self.channels_out[channel_output_index] = (
                        data_input[map_value]
                    )
                except Exception as e:
                    print(
                        (
                            "additional info:\n" +
                            "  channel_output_index: {}\n" +
                            "  len(self.channels_out): {}\n" +
                            "  map_value: {}\n"
                        ).format(
                            channel_output_index,
                            len(self.channels_out),
                            map_value
                        )
                    )
                    raise
            else:
                # don't alter data
                pass

        self.dmx_send_frame(
            self.config['universe']['output'],
            self.channels_out
        )

##########################################
if __name__ == '__main__':

    print(42*'*')
    print('Python Version: ' + sys.version)
    print(42*'*')
    print(__doc__)
    print(42*'*')

    # parse arguments
    filename = "map.json"
    # only use args after script name
    arg = sys.argv[1:]
    if not arg:
        print("using standard values.")
        print(" Allowed parameters:")
        print("   filename for config file       (default='map.json')")
        print("")
    else:
        filename = arg[0]
        # if len(arg) > 1:
        #     pixel_count = int(arg[1])
    # print parsed argument values
    print('''values:
        filename :{}
    '''.format(filename))

    default_config = {
        'universe': {
            'input': 1,
            'output': 2,
            'channel_count': 240,
        },
        'map': {
            'channels': [
                -1,
                -1,
                -1,
                -1,
                30,
                31,
                -1,
                -1,
                -1,
                -1,
                22,
                23,
                -1,
                -1,
                -1,
                -1,
                14,
                15,
                -1,
                -1,
                -1,
                -1,
                6,
                7,
                28,
                29,
                26,
                27,
                24,
                25,
                20,
                21,
                18,
                19,
                16,
                17,
                12,
                13,
                10,
                11,
                8,
                9,
                4,
                5,
                2,
                3,
                0,
                1,
            ],
            'repeat': 5,
            'repeat_reverse': True,
            'offset': True,
            'offset_count': 32,
        },
    }
    my_config = ConfigDict(default_config, filename)
    print("my_config.config: {}".format(my_config.config))

    my_mapper = OLAMapper(my_config.config)
    print("full map:\n{}".format(my_mapper.map_tostring_pretty()))

    my_mapper.start_ola()

    # wait for user to hit key.
    try:
        raw_input(
            "\n\n" +
            42*'*' +
            "\nhit a key to stop the mapper\n" +
            42*'*' +
            "\n\n"
        )
    except KeyboardInterrupt:
        print("\nstop.")
    except:
        print("\nstop.")

    # blocks untill thread has joined.
    my_mapper.stop_ola()

    my_mapper.print_measurements()

    # as last thing we save the current configuration.
    print("\nwrite config.")
    my_config.write_to_file()

    # ###########################################
<|endoftext|>


<commit_before># ome - Object Message Expressions
# Copyright (c) 2015-2016 Luke McCarthy <luke@iogopro.co.uk>

import re
import subprocess
import time
from .error import OmeError
from .target import target_map
from .types import CompileOptions

def get_target(target_name):
    target_name = target_name.lower()
    if target_name not in target_map:
        raise OmeError("unknown target '{}'".format(target_name))
    return target_map[target_name]

def get_backend_version(backend):
    if not hasattr(backend, 'version') and hasattr(backend, 'command'):
        reason = 'could not get version number'
        args = [backend.command] + backend.version_args
        try:
            process = subprocess.Popen(args, stdout=subprocess.PIPE)
            outs, errs = process.communicate()
            if process.returncode == 0:
                m = re.match(backend.version_re, outs.decode('ascii'))
                if m:
                    backend.version = m.group(1)
                    return
        except OSError as e:
            reason = str(e)
        except UnicodeDecodeError:
            pass
        raise OmeError("backend '{}' is not available: {}".format(backend.name, reason))

def get_backend(target, platform, backend_name=None, backend_command=None):
    platform = platform.lower()
    if backend_name is None:
        for backend_name in target.backend_preference:
            backend = target.backends[backend_name]
            backend = backend(backend_command or backend.default_command)
            if not hasattr(backend, 'supported_platforms') or platform in backend.supported_platforms:
                try:
                    get_backend_version(backend)
                    return backend
                except OmeError:
                    pass
        raise OmeError("could not find a working backend for target '{}' for platform '{}'".format(target.name, platform))
    else:
        backend_name = backend_name.lower()
        if backend_name not in target.backends:
            raise OmeError("unknown backend '{}' for target '{}'".format(backend_name, target.name))
        backend = target.backends[backend_name]
        backend = backend(backend_command or backend.default_command)
        get_backend_version(backend)
        return backend

def run_shell_command(args, input=None, output=None):
    process = subprocess.Popen(args,
        stdin = input and subprocess.PIPE,
        stdout = output and subprocess.PIPE)
    outs, errs = process.communicate(input)
    if process.returncode != 0:
        raise OmeError('command failed with return code {}'.format(process.returncode))
    return outs

def get_args_list(args):
    if len(args) == 1 and isinstance(args[0], (list, tuple)):
        args = args[0]
    return args

class BuildShell(object):
    def __init__(self, show_commands=False):
        self.show_commands = show_commands

    def print_command(self, args):
        if self.show_commands:
            print(' '.join(args))

    def run(self, *args, input=None):
        args = get_args_list(args)
        self.print_command(args)
        run_shell_command(args, input)

    def run_output(self, *args, input=None):
        args = get_args_list(args)
        self.print_command(args)
        return run_shell_command(args, input, True)

platform_aliases = {
    'linux': ['posix'],
}

class BuildOptions(CompileOptions):
    def __init__(self, target, options):
        self.target = target
        self.platform = options.platform.lower()
        self.debug = options.debug
        self.link = not options.make_object
        self.static = options.static
        self.verbose = options.verbose
        self.verbose_backend = options.verbose_backend
        self.traceback = not options.no_traceback
        self.source_traceback = not options.no_source_traceback
        self.use_musl = options.use_musl
        self.musl_path = options.musl_path
        self.shell = BuildShell(options.show_build_commands)
        self.include_dirs = []
        self.lib_dirs = []
        self.dynamic_libs = []
        self.static_libs = []
        self.defines = [
            ('OME_PLATFORM', self.platform),
            ('OME_PLATFORM_' + self.platform.upper(), ''),
        ]
        for platform_alias in platform_aliases.get(self.platform):
            self.defines.append(('OME_PLATFORM_' + platform_alias.upper(), ''))
        if not options.debug:
            self.defines.append(('NDEBUG', ''))
        if options.debug_gc:
            self.defines.append(('OME_GC_DEBUG', ''))
        if options.gc_stats:
            self.defines.append(('OME_GC_STATS', ''))
        if options.no_traceback:
            self.defines.append(('OME_NO_TRACEBACK', ''))
        if options.no_source_traceback:
            self.defines.append(('OME_NO_SOURCE_TRACEBACK', ''))

    def make_output(self, input, outfile, backend):
        if hasattr(backend, 'supported_platforms') and self.platform not in backend.supported_platforms:
            raise OmeError("backend '{}' does not support platform '{}'".format(backend.name, self.platform))
        backend.make_output(self.shell, input, outfile, self)
<commit_msg>Removed run_output().<commit_after># ome - Object Message Expressions
# Copyright (c) 2015-2016 Luke McCarthy <luke@iogopro.co.uk>

import re
import subprocess
import time
from .error import OmeError
from .target import target_map
from .types import CompileOptions

def get_target(target_name):
    target_name = target_name.lower()
    if target_name not in target_map:
        raise OmeError("unknown target '{}'".format(target_name))
    return target_map[target_name]

def get_backend_version(backend):
    if not hasattr(backend, 'version') and hasattr(backend, 'command'):
        reason = 'could not get version number'
        args = [backend.command] + backend.version_args
        try:
            process = subprocess.Popen(args, stdout=subprocess.PIPE)
            outs, errs = process.communicate()
            if process.returncode == 0:
                m = re.match(backend.version_re, outs.decode('ascii'))
                if m:
                    backend.version = m.group(1)
                    return
        except OSError as e:
            reason = str(e)
        except UnicodeDecodeError:
            pass
        raise OmeError("backend '{}' is not available: {}".format(backend.name, reason))

def get_backend(target, platform, backend_name=None, backend_command=None):
    platform = platform.lower()
    if backend_name is None:
        for backend_name in target.backend_preference:
            backend = target.backends[backend_name]
            backend = backend(backend_command or backend.default_command)
            if not hasattr(backend, 'supported_platforms') or platform in backend.supported_platforms:
                try:
                    get_backend_version(backend)
                    return backend
                except OmeError:
                    pass
        raise OmeError("could not find a working backend for target '{}' for platform '{}'".format(target.name, platform))
    else:
        backend_name = backend_name.lower()
        if backend_name not in target.backends:
            raise OmeError("unknown backend '{}' for target '{}'".format(backend_name, target.name))
        backend = target.backends[backend_name]
        backend = backend(backend_command or backend.default_command)
        get_backend_version(backend)
        return backend

def run_shell_command(args, input=None, output=None):
    process = subprocess.Popen(args,
        stdin = input and subprocess.PIPE,
        stdout = output and subprocess.PIPE)
    outs, errs = process.communicate(input)
    if process.returncode != 0:
        raise OmeError('command failed with return code {}'.format(process.returncode))
    return outs

def get_args_list(args):
    if len(args) == 1 and isinstance(args[0], (list, tuple)):
        args = args[0]
    return args

class BuildShell(object):
    def __init__(self, show_commands=False):
        self.show_commands = show_commands

    def print_command(self, args):
        if self.show_commands:
            print(' '.join(args))

    def run(self, *args, input=None, output=None):
        args = get_args_list(args)
        self.print_command(args)
        return run_shell_command(args, input, output)

platform_aliases = {
    'linux': ['posix'],
}

class BuildOptions(CompileOptions):
    def __init__(self, target, options):
        self.target = target
        self.platform = options.platform.lower()
        self.debug = options.debug
        self.link = not options.make_object
        self.static = options.static
        self.verbose = options.verbose
        self.verbose_backend = options.verbose_backend
        self.traceback = not options.no_traceback
        self.source_traceback = not options.no_source_traceback
        self.use_musl = options.use_musl
        self.musl_path = options.musl_path
        self.shell = BuildShell(options.show_build_commands)
        self.include_dirs = []
        self.lib_dirs = []
        self.dynamic_libs = []
        self.static_libs = []
        self.defines = [
            ('OME_PLATFORM', self.platform),
            ('OME_PLATFORM_' + self.platform.upper(), ''),
        ]
        for platform_alias in platform_aliases.get(self.platform):
            self.defines.append(('OME_PLATFORM_' + platform_alias.upper(), ''))
        if not options.debug:
            self.defines.append(('NDEBUG', ''))
        if options.debug_gc:
            self.defines.append(('OME_GC_DEBUG', ''))
        if options.gc_stats:
            self.defines.append(('OME_GC_STATS', ''))
        if options.no_traceback:
            self.defines.append(('OME_NO_TRACEBACK', ''))
        if options.no_source_traceback:
            self.defines.append(('OME_NO_SOURCE_TRACEBACK', ''))

    def make_output(self, input, outfile, backend):
        if hasattr(backend, 'supported_platforms') and self.platform not in backend.supported_platforms:
            raise OmeError("backend '{}' does not support platform '{}'".format(backend.name, self.platform))
        backend.make_output(self.shell, input, outfile, self)
<|endoftext|>


<commit_before>import re

import yaml
from odict import OrderedDict

from item import Elem, Item

# TODO Move into real config? Make user-configurable?
DATA_FILE = "orgm.dat"

class OrganizemIllegalUsageException(Exception): pass

class Action(object):
    ADD = 'add'
    ADD_EMPTY = 'add_empty'
    REMOVE = 'remove'
    FIND = 'find'
    SHOW_GROUPED = 'show_grouped'
    SHOW_ELEMENTS = 'show_elements'
    REBUILD_GROUPED = 'rebuild_grouped'
    BACKUP = 'backup'

class ActionArg(object):
    REGEX = 'regex'
    FILENAME = 'filename'
    BY_TITLE = 'by_title'
    BY_AREA = 'by_area'
    BY_PROJECT = 'by_project'
    BY_TAGS = 'by_tags'
    BY_ACTIONS = 'by_actions'

class Organizem(object):
    """
    Manages an Organizem TODO data file, binding to a data file by name
    and exposing methods that implement the Actions enum-ed above
    in class Actions.
    """

    # Public API
    # Testers, extenders or anyone who wants to use Organizem programatically
    #  can call these public methods directly. Regular usage is from commmand line
    #  which only uses #run_cli()
    def __init__(self, data_file=DATA_FILE):
        self.data_file = data_file
        self.data = []

    # Just appends item to end of file using Item.__str__()
    # Maybe there is a way to leverage the library to yaml-ize transparently?
    def add_item(self, item):
        with open(self.data_file, 'a') as f:
            f.write(str(item))

    def add_empty(self):
        self.add_item(Item(''))

    # NOTE: Finds all matching items, trying to match the element, value passed
    #  in, e.g. TITLE and a title value, PROJECT and a project value, etc.
    # NOTE: element must be one of the "enums" in class Element (other than ROOT)
    # NOTE: Returns list of 0 or more Items
    def find_items(self, element, pattern, is_regex_match=False):
        return self._find_or_filter_items(element, pattern, is_regex_match, is_filter=False)

    def remove_items(self, element, pattern, is_regex_match=False):
        # Call helper with filtering on and filter predicate the pattern passed in
        # Result is we get back (all rows) - (those that match)
        filtered_items = self._find_or_filter_items(element, pattern, is_regex_match, is_filter=True)
        # Send all the rewrites to the helper to create new data file
        self._rewrite(filtered_items)

    def get_elements(self, element):
        self.data = self._load()
        # Use set to get unique values only
        ret = set()
        if self.data is None:
            return ret

        for item in self.data:
            # Skip empty string and empty list, not interesting to return anyway
            # Handle case of list elements, loop over those to get their values
            val = item.get_elem_val(element)
            if val and len(val):
                if isinstance(val, str):
                    ret.add(val)
                elif isinstance(val, list):
                    for v in val:
                        ret.add(v)
        # Now convert the set of uniqe values to list to sort it for return. Return contract is
        #  is sorted in descending order.
        ret = list(ret)
        ret.sort()
        return ret

    # Groups all items by the distinct values in the element passed in, e.g.
    #  groups by all the tags in the file, or projects or areas.
    def get_grouped_items(self, element):
        self.data = self._load()
        # Use dictionary that keeps keys in sorted order to return items sorted
        #  by the values for the element key that is being used to group the items
        # NOTE: This is 3p module, odict.py
        ret = OrderedDict()
        if self.data is None:
            return ret

        for item in self.data:
            group_keys = item.get_elem_val(element)
            if isinstance(group_keys, str):
                group_keys = [group_keys]
            for group_key in group_keys:
                if group_key not in ret:
                    ret[group_key] = []
                ret[group_key].append(item)
        return ret

    # Groups items as get_grouped_items() does, then backs up the current
    #  data file to [file_name]_bak, writes the grouped items to the file
    #  and also returns them for convenience and testing purposes
    def regroup_data_file(self, element):
        grouped_items = self.get_grouped_items(element)
        items = []
        if grouped_items is None:
            return ""
        for group_key in grouped_items.keys():
            for item in grouped_items[group_key]:
                # Call item util method to convert PyItem returned by get_grouped_items()
                #  to Item that can be converted to YAML by self._rewrite()
                items.append(item)
        # Pass the Items, regrouped, to be written to data file
        self._rewrite(items)
        # Now as conveniennce and for debugging, pass the same output back to stdout
        ret = []
        for item in items:
            ret.append(str(item))
        return "\n".join(ret)

    def backup(self, bak_data_file=None):
        if not bak_data_file:
            bak_data_file = self.data_file + '_bak'
        self._backup(bak_data_file)

    # CLI API
    # Command line users use only this call, indirectly, by passing command lines
    #  to orgm.py#__main__()
    def run_cli(self, title, args):
        # Get the action to be taken
        action = args.action
        # Get the mandatory title Element
        title = args.title
        # Get any optional Element values
        area = args.area
        project = args.project
        # Must split() the two list element types, so the string becomes Py list
        tags = args.tags.split(',')
        actions = args.actions.split(',')
        due_date = args.due_date
        note = args.note
        # Get optional Modifiers, regex, filename
        is_regex_match = args.regex
        # Trim single- or double-quotes from filename -- annoying to have this in name
        #  but CLI everywhere else has quotes around args, so this means user
        #  doesn't need to remember to do that
        filename = args.filename
        if filename and len(filename):
            first_char = filename[0]
            # TODO This is obviously flimsy (can have different style quote at each end)
            last_char = filename[len(filename) - 1]
            if ((first_char == "'" or first_char == '"') and \
                (last_char == "'" or last_char == '"')):
                filename = filename[1:-1]
        # group_by_* Modifiers
        # Required for --show_grouped, --rebuild_grouped, --show_elements
        # TODO add validation
        by_title = args.by_title
        by_area = args.by_area
        by_project = args.by_project
        by_tags = args.by_tags
        by_actions = args.by_actions

        # For actions matching on an element value, figure out which one
        # NOTE: Just uses first one that by usine if/elif.  DOES NOT
        #  validate only one was passed in and raise Exception
        match_elem = None
        match_val = None
        if action == Action.FIND or action == Action.REMOVE:
            if len(title):
                match_elem = Elem.TITLE
                match_val = title
            elif len(tags):
                match_elem = Elem.TAGS
                match_val = tags
            elif len(actions):
                match_elem = Elem.ACTIONS
                match_val = actions
            elif len(note):
                match_elem = Elem.NOTE
                match_val = note

        # For actions modified by a group_by arg, figure out which one
        group_elem = None
        if action == Action.SHOW_GROUPED or \
            action == Action.REBUILD_GROUPED or \
            action == Action.SHOW_ELEMENTS:
            if by_title:
                group_elem = Elem.TITLE
            elif by_area:
                group_elem = Elem.AREA
            elif by_project:
                group_elem = Elem.PROJECT
            elif by_tags:
                group_elem = Elem.TAGS
            elif by_actions:
                group_elem = Elem.ACTIONS

        # Now turn cmd line action and arguments into Organizem API call
        if action == Action.ADD:
            if len(title) == 0:
                raise OrganizemIllegalUsageException("'--add' action must include '--title' element and a value for title.")
            item = Item(title, \
                area=area, project=project, tags=tags, \
                actions=actions, due_date=due_date, \
                note=note)
            self.add_item(item)

        elif action == Action.ADD_EMPTY:
            self.add_empty()

        elif action == Action.REMOVE:
            self.remove_items(match_elem, match_val, is_regex_match)

        elif action == Action.FIND:
            items = self.find_items(match_elem, match_val, is_regex_match)
            for item in items:
                print str(item)

        elif action == Action.SHOW_GROUPED:
            grouped_items = self.get_grouped_items(group_elem)
            group_keys = grouped_items.keys()
            group_keys.sort()
            for group_key in group_keys:
                print '\n\n*** ' + group_key + ' ***'
                for item in grouped_items[group_key]:
                    print str(item)

        elif action == Action.SHOW_ELEMENTS:
            elems = self.get_elements(group_elem)
            elems.sort()
            for elem in elems:
                print elem

        # TODO
        elif action == Action.REBUILD_GROUPED:
            self.regroup_data_file(group_elem)

        elif action == Action.BACKUP:
            self.backup(filename)

    # Helpers

    # This is just to avoid code duplication.  Loops over elements and matches predicate
    #  that is matching pattern.  Applies regex match or not, based on flag, and either
    #  includes matches (if is_filter=False) or excludes matches (if is_filter=True).  Used by
    #  both find_items() and remove_items(), with remove_items() having is_filter=True
    def _find_or_filter_items(self, element, pattern, is_regex_match, is_filter):
        ret = []
        self.data = self._load()
        for item in self.data:
            match_val = item.get_elem_val(element)

            # Support case that some elements are str, and some are list
            # These are the match cases:
            # 1) pattern is str, match_val is str, not is_filter
            # 2) pattern is str, match_val is str, is_filter
            # 3) pattern is str, match_val is list, not is_filter
            # 4) pattern is str, match_val is list, is_filter
            # 5) pattern is list, match_val is str, not is_filter
            # 6) pattern is list, match_val is str, is_filter
            # 7) pattern is list, match_val is list, not is_filter
            # 8) pattern is list, match_val is list, is_filter

            # 1) and 2)
            # Straight match scalar value to scalar value
            if isinstance(pattern, str) and isinstance(match_val, str):
                is_match = self._is_match(pattern, match_val, element, is_regex_match)
                # 1)
                if (is_match and not is_filter):
                    ret.append(item)
                # 2)
                elif(not is_match and is_filter):
                    ret.append(item)
            # 3) and 4)
            # Match scalar pattern to any element of elem_val list
            elif isinstance(pattern, str) and isinstance(match_val, list):
                # Scope outside loop because we only include this item in is_filter case
                #  if we've checked every item and *none* match
                is_match = False
                for v in match_val:
                    is_match = self._is_match(pattern, v, element, is_regex_match)
                     # 3)
                     # Matched and we aren't filtering, so first match means we can short-circuit
                    if (is_match and not is_filter):
                        ret.append(item)
                        break
                # 4)
                # Filtering, we checked every val in elem_val and none matched pattern, so include item
                if is_filter and not is_match:
                    ret.append(item)
            # 5) and 6)
            # Match any element in list pattern to scalar elem_val
            elif isinstance(pattern, list) and isinstance(match_val, str):
                is_match = False
                for p in pattern:
                    is_match = self._is_match(p, match_val, element, is_regex_match)
                     # 5)
                     # Matched and we aren't filtering, so first match means we can short-circuit
                    if (is_match and not is_filter):
                        ret.append(item)
                        break
                # 6)
                # Filtering, we checked every p in pattern and none matched elem_val, so include item
                if is_filter and not is_match:
                    ret.append(item)
            # 7) and 8)
            # Match any element in list pattern to any elmentn in list elem_val
            elif isinstance(pattern, list) and isinstance(match_val, list):
                is_match = False
                for p in pattern:
                    for v in match_val:
                        is_match = self._is_match(p, v, element, is_regex_match)
                        # 7)
                        # Matched and we aren't filtering, so first match means we can short-circuit
                        if (is_match and not is_filter):
                            ret.append(item)
                            break
                    # Now break out of outer loop in case of match and not filter, second short-circuit
                    if (is_match and not is_filter):
                        break
                # 8)
                # Filtering, we checked every p in pattern and none matched elem_val, so include item
                if is_filter and not is_match:
                    ret.append(item)
        # If no matches this returns empty list, which evaluates to "False"
        return ret

    def _is_match(self, pattern, match_val, element, is_regex_match):
        if not is_regex_match:
            return pattern == match_val
        else:
            # Note types can have line breaks and lots of crap.
            # Increase our chances of avoiding trouble by removing line breaks
            if element == Elem.NOTE:
               match_val = match_val.replace('\n', ' ')
            rgx = re.compile(pattern, re.IGNORECASE)
            return rgx.search(match_val) != None

    def _load(self):
        with open(self.data_file) as f:
            # NOTE: PyYaml returns list/dict hybrid deserialization from YAML, not class Item
            #  objects we want to manipulate throughout our code (to have one standard, clean, OO
            #  representation of an Item).  So we convert her on _load() and then use Items everywhere
            py_items = yaml.load(f)
            if py_items and len(py_items):
                # We got back to data to load, clear any previous state in self.data
                del self.data[:]
                for py_item in py_items:
                    # Convert each item retrieved from the file to a class Item object
                    self.data.append(Item.init_from_py_item(py_item))
        return self.data

    # Useful for debugging, though doesn't support any current feature
    def _dump(self):
        with open(self.data_file, 'r') as f:
            lines = f.readlines()
            for line in lines:
                print line

    # Utility and used by regroup_data_file
    def _backup(self, bak_data_file):
        with open(self.data_file, 'r') as fr:
            lines = fr.readlines()
            with open(bak_data_file, 'w') as fw:
                for line in lines:
                    fw.write(line)

    def _rewrite(self, items):
        self._backup(self.data_file + '_bak')
        with open(self.data_file, 'w') as f:
            for item in items:
                f.write(str(item))

<commit_msg>Minor Pythonic fixes from review feedback<commit_after>import re

import yaml
from odict import OrderedDict

from item import Elem, Item

# TODO Move into real config? Make user-configurable?
DATA_FILE = "orgm.dat"

class OrganizemIllegalUsageException(Exception): pass

class Action(object):
    ADD = 'add'
    ADD_EMPTY = 'add_empty'
    REMOVE = 'remove'
    FIND = 'find'
    SHOW_GROUPED = 'show_grouped'
    SHOW_ELEMENTS = 'show_elements'
    REBUILD_GROUPED = 'rebuild_grouped'
    BACKUP = 'backup'

class ActionArg(object):
    REGEX = 'regex'
    FILENAME = 'filename'
    BY_TITLE = 'by_title'
    BY_AREA = 'by_area'
    BY_PROJECT = 'by_project'
    BY_TAGS = 'by_tags'
    BY_ACTIONS = 'by_actions'

class Organizem(object):
    """
    Manages an Organizem TODO data file, binding to a data file by name
    and exposing methods that implement the Actions enum-ed above
    in class Actions.
    """

    # Public API
    # Testers, extenders or anyone who wants to use Organizem programatically
    #  can call these public methods directly. Regular usage is from commmand line
    #  which only uses #run_cli()
    def __init__(self, data_file=DATA_FILE):
        self.data_file = data_file
        self.data = []

    # Just appends item to end of file using Item.__str__()
    # Maybe there is a way to leverage the library to yaml-ize transparently?
    def add_item(self, item):
        with open(self.data_file, 'a') as f:
            f.write(str(item))

    def add_empty(self):
        self.add_item(Item(''))

    # NOTE: Finds all matching items, trying to match the element, value passed
    #  in, e.g. TITLE and a title value, PROJECT and a project value, etc.
    # NOTE: element must be one of the "enums" in class Element (other than ROOT)
    # NOTE: Returns list of 0 or more Items
    def find_items(self, element, pattern, is_regex_match=False):
        return self._find_or_filter_items(element, pattern, is_regex_match, is_filter=False)

    def remove_items(self, element, pattern, is_regex_match=False):
        # Call helper with filtering on and filter predicate the pattern passed in
        # Result is we get back (all rows) - (those that match)
        filtered_items = self._find_or_filter_items(element, pattern, is_regex_match, is_filter=True)
        # Send all the rewrites to the helper to create new data file
        self._rewrite(filtered_items)

    def get_elements(self, element):
        self.data = self._load()
        # Use set to get unique values only
        ret = set()
        if self.data is None:
            return ret

        for item in self.data:
            # Skip empty string and empty list, not interesting to return anyway
            # Handle case of list elements, loop over those to get their values
            val = item.get_elem_val(element)
            if val and len(val):
                if isinstance(val, str):
                    ret.add(val)
                elif isinstance(val, list):
                    for v in val:
                        ret.add(v)
        # Now convert the set of uniqe values to list to sort it for return. Return contract is
        #  is sorted in descending order.
        ret = list(ret)
        ret.sort()
        return ret

    # Groups all items by the distinct values in the element passed in, e.g.
    #  groups by all the tags in the file, or projects or areas.
    def get_grouped_items(self, element):
        self.data = self._load()
        # Use dictionary that keeps keys in sorted order to return items sorted
        #  by the values for the element key that is being used to group the items
        # NOTE: This is 3p module, odict.py
        ret = OrderedDict()
        if self.data is None:
            return ret

        for item in self.data:
            group_keys = item.get_elem_val(element)
            if isinstance(group_keys, str):
                group_keys = [group_keys]
            for group_key in group_keys:
                if group_key not in ret:
                    ret[group_key] = []
                ret[group_key].append(item)
        return ret

    # Groups items as get_grouped_items() does, then backs up the current
    #  data file to [file_name]_bak, writes the grouped items to the file
    #  and also returns them for convenience and testing purposes
    def regroup_data_file(self, element):
        grouped_items = self.get_grouped_items(element)
        items = []
        if grouped_items is None:
            return ""
        for group_key in grouped_items.keys():
            items.extend(grouped_items[group_key])
        # Pass the Items, regrouped, to be written to data file
        self._rewrite(items)
        # For conveninence, testing/debugging pass the same output back to stdout
        return "\n".join([str(item) for item in items])

    def backup(self, bak_data_file=None):
        if not bak_data_file:
            bak_data_file = self.data_file + '_bak'
        self._backup(bak_data_file)

    # CLI API
    # Command line users use only this call, indirectly, by passing command lines
    #  to orgm.py#__main__()
    def run_cli(self, title, args):
        # Get the action to be taken
        action = args.action
        # Get the mandatory title Element
        title = args.title
        # Get any optional Element values
        area = args.area
        project = args.project
        # Must split() the two list element types, so the string becomes Py list
        tags = args.tags.split(',')
        actions = args.actions.split(',')
        due_date = args.due_date
        note = args.note
        # Get optional Modifiers, regex, filename
        is_regex_match = args.regex
        # Trim single- or double-quotes from filename -- annoying to have this in name
        #  but CLI everywhere else has quotes around args, so this means user
        #  doesn't need to remember to do that
        filename = args.filename
        if filename and len(filename):
            first_char = filename[0]
            # TODO This is obviously flimsy (can have different style quote at each end)
            last_char = filename[len(filename) - 1]
            if ((first_char == "'" or first_char == '"') and \
                (last_char == "'" or last_char == '"')):
                filename = filename[1:-1]
        # group_by_* Modifiers
        # Required for --show_grouped, --rebuild_grouped, --show_elements
        # TODO add validation
        by_title = args.by_title
        by_area = args.by_area
        by_project = args.by_project
        by_tags = args.by_tags
        by_actions = args.by_actions

        # For actions matching on an element value, figure out which one
        # NOTE: Just uses first one that by usine if/elif.  DOES NOT
        #  validate only one was passed in and raise Exception
        match_elem = None
        match_val = None
        if action == Action.FIND or action == Action.REMOVE:
            if len(title):
                match_elem = Elem.TITLE
                match_val = title
            elif len(tags):
                match_elem = Elem.TAGS
                match_val = tags
            elif len(actions):
                match_elem = Elem.ACTIONS
                match_val = actions
            elif len(note):
                match_elem = Elem.NOTE
                match_val = note

        # For actions modified by a group_by arg, figure out which one
        group_elem = None
        if action == Action.SHOW_GROUPED or \
            action == Action.REBUILD_GROUPED or \
            action == Action.SHOW_ELEMENTS:
            if by_title:
                group_elem = Elem.TITLE
            elif by_area:
                group_elem = Elem.AREA
            elif by_project:
                group_elem = Elem.PROJECT
            elif by_tags:
                group_elem = Elem.TAGS
            elif by_actions:
                group_elem = Elem.ACTIONS

        # Now turn cmd line action and arguments into Organizem API call
        if action == Action.ADD:
            if len(title) == 0:
                raise OrganizemIllegalUsageException("'--add' action must include '--title' element and a value for title.")
            item = Item(title, \
                area=area, project=project, tags=tags, \
                actions=actions, due_date=due_date, \
                note=note)
            self.add_item(item)

        elif action == Action.ADD_EMPTY:
            self.add_empty()

        elif action == Action.REMOVE:
            self.remove_items(match_elem, match_val, is_regex_match)

        elif action == Action.FIND:
            items = self.find_items(match_elem, match_val, is_regex_match)
            for item in items:
                print str(item)

        elif action == Action.SHOW_GROUPED:
            grouped_items = self.get_grouped_items(group_elem)
            group_keys = grouped_items.keys()
            group_keys.sort()
            for group_key in group_keys:
                print '\n\n*** ' + group_key + ' ***'
                for item in grouped_items[group_key]:
                    print str(item)

        elif action == Action.SHOW_ELEMENTS:
            elems = self.get_elements(group_elem)
            elems.sort()
            for elem in elems:
                print elem

        elif action == Action.REBUILD_GROUPED:
            self.regroup_data_file(group_elem)

        elif action == Action.BACKUP:
            self.backup(filename)

    # Helpers

    # This is just to avoid code duplication.  Loops over elements and matches predicate
    #  that is matching pattern.  Applies regex match or not, based on flag, and either
    #  includes matches (if is_filter=False) or excludes matches (if is_filter=True).  Used by
    #  both find_items() and remove_items(), with remove_items() having is_filter=True
    def _find_or_filter_items(self, element, pattern, is_regex_match, is_filter):
        ret = []
        self.data = self._load()
        for item in self.data:
            match_val = item.get_elem_val(element)

            # Support case that some elements are str, and some are list
            # These are the match cases:
            # 1) pattern is str, match_val is str, not is_filter
            # 2) pattern is str, match_val is str, is_filter
            # 3) pattern is str, match_val is list, not is_filter
            # 4) pattern is str, match_val is list, is_filter
            # 5) pattern is list, match_val is str, not is_filter
            # 6) pattern is list, match_val is str, is_filter
            # 7) pattern is list, match_val is list, not is_filter
            # 8) pattern is list, match_val is list, is_filter

            # 1) and 2)
            # Straight match scalar value to scalar value
            if isinstance(pattern, str) and isinstance(match_val, str):
                is_match = self._is_match(pattern, match_val, element, is_regex_match)
                # 1)
                if (is_match and not is_filter):
                    ret.append(item)
                # 2)
                elif(not is_match and is_filter):
                    ret.append(item)
            # 3) and 4)
            # Match scalar pattern to any element of elem_val list
            elif isinstance(pattern, str) and isinstance(match_val, list):
                # Scope outside loop because we only include this item in is_filter case
                #  if we've checked every item and *none* match
                is_match = False
                for v in match_val:
                    is_match = self._is_match(pattern, v, element, is_regex_match)
                     # 3)
                     # Matched and we aren't filtering, so first match means we can short-circuit
                    if (is_match and not is_filter):
                        ret.append(item)
                        break
                # 4)
                # Filtering, we checked every val in elem_val and none matched pattern, so include item
                if is_filter and not is_match:
                    ret.append(item)
            # 5) and 6)
            # Match any element in list pattern to scalar elem_val
            elif isinstance(pattern, list) and isinstance(match_val, str):
                is_match = False
                for p in pattern:
                    is_match = self._is_match(p, match_val, element, is_regex_match)
                     # 5)
                     # Matched and we aren't filtering, so first match means we can short-circuit
                    if (is_match and not is_filter):
                        ret.append(item)
                        break
                # 6)
                # Filtering, we checked every p in pattern and none matched elem_val, so include item
                if is_filter and not is_match:
                    ret.append(item)
            # 7) and 8)
            # Match any element in list pattern to any elmentn in list elem_val
            elif isinstance(pattern, list) and isinstance(match_val, list):
                is_match = False
                for p in pattern:
                    for v in match_val:
                        is_match = self._is_match(p, v, element, is_regex_match)
                        # 7)
                        # Matched and we aren't filtering, so first match means we can short-circuit
                        if (is_match and not is_filter):
                            ret.append(item)
                            break
                    # Now break out of outer loop in case of match and not filter, second short-circuit
                    if (is_match and not is_filter):
                        break
                # 8)
                # Filtering, we checked every p in pattern and none matched elem_val, so include item
                if is_filter and not is_match:
                    ret.append(item)
        # If no matches this returns empty list, which evaluates to "False"
        return ret

    def _is_match(self, pattern, match_val, element, is_regex_match):
        if not is_regex_match:
            return pattern == match_val
        else:
            # Note types can have line breaks and lots of crap.
            # Increase our chances of avoiding trouble by removing line breaks
            if element == Elem.NOTE:
               match_val = match_val.replace('\n', ' ')
            rgx = re.compile(pattern, re.IGNORECASE)
            return rgx.search(match_val) != None

    def _load(self):
        with open(self.data_file) as f:
            # NOTE: PyYaml returns list/dict hybrid deserialization from YAML, not class Item
            #  objects we want to manipulate throughout our code (to have one standard, clean, OO
            #  representation of an Item).  So we convert her on _load() and then use Items everywhere
            py_items = yaml.load(f)
            if py_items and len(py_items):
                # We got back to data to load, clear any previous state in self.data
                del self.data[:]
                for py_item in py_items:
                    # Convert each item retrieved from the file to a class Item object
                    self.data.append(Item.init_from_py_item(py_item))
        return self.data

    # Useful for debugging, though doesn't support any current feature
    def _dump(self):
        with open(self.data_file, 'r') as f:
            for line in f:
                print line

    def _backup(self, bak_data_file):
  	    import shutil
  	    shutil.copyfile(self.data_file, bak_data_file)

    def _rewrite(self, items):
        self._backup(self.data_file + '_bak')
        with open(self.data_file, 'w') as f:
            for item in items:
                f.write(str(item))
<|endoftext|>


<commit_before>"""DWC Network Server Emulator

    Copyright (C) 2014 polaris-
    Copyright (C) 2014 ToadKing
    Copyright (C) 2016 Sepalani

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import os
import random
import time

from gamespy.gs_database import GamespyDatabase

# If a game from this list requests a file listing, the server will return
# that only one exists and return a random one.
# This is used for Mystery Gift distribution on Generation 4 Pokemon games
gamecodes_return_random_file = [
    'ADAD',
    'ADAE',
    'ADAF',
    'ADAI',
    'ADAJ',
    'ADAK',
    'ADAS',
    'CPUD',
    'CPUE',
    'CPUF',
    'CPUI',
    'CPUJ',
    'CPUK',
    'CPUS',
    'IPGD',
    'IPGE',
    'IPGF',
    'IPGI',
    'IPGJ',
    'IPGK',
    'IPGS'
]

def get_file_count(data):
    return sum(1 for line in data.splitlines() if line)

def filter_list(data, attr1=None, attr2=None, attr3=None,
                num=None, offset=None):
    """Filter the list based on the attribute fields.

    If nothing matches, at least return a newline.
    Pokemon BW at least expects this and will error without it.
    """
    if attr1 is None and attr2 is None and attr3 is None and \
       num is None and offset is None:
        # Nothing to filter, just return the input data
        return data

    def attrs(data):
        """Filter attrs."""
        def nc(a, b):
            """Filter nc."""
            return a is None or a == b
        return \
            len(data) == 6 and \
            nc(attr1, data[2]) and \
            nc(attr2, data[3]) and \
            nc(attr3, data[4])
    output = filter(lambda line: attrs(line.split("\t")), data.splitlines())

    if offset is not None:
        output = output[offset:]

    if num is not None:
        output = output[:num]

    return '\r\n'.join(output) + '\r\n'

def filter_list_random_files(data, count):
    """Get [count] random files from the filelist."""
    samples = random.sample(data.splitlines(), count)
    return '\r\n'.join(samples) + '\r\n'

def filter_list_by_date(data, token):
    """Allow user to control which file to receive by setting
    the local date selected file will be the one at
    index (day of year) mod (file count)."""
    try:
        userData = GamespyDatabase().get_nas_login(token)
        date = time.strptime(userData['devtime'], '%y%m%d%H%M%S')
        files = data.splitlines()
        ret = files[(int(date.tm_yday) - 1) % len(files)] + '\r\n'
    except:
        ret = filter_list_random_files(data, 1)
    return ret

def filter_list_g5_mystery_gift(data, rhgamecd):
    """Custom selection for generation 5 mystery gifts, so that the random
    or data-based selection still works properly."""
    if rhgamecd[2] == 'A':
        filterBit = 0x100000
    elif rhgamecd[2] == 'B':
        filterBit = 0x200000
    elif rhgamecd[2] == 'D':
        filterBit = 0x400000
    elif rhgamecd[2] == 'E':
        filterBit = 0x800000
    else:
        # unknown game, can't filter
        return data

    output = []
    for line in data.splitlines():
        lineBits = int(line.split('\t')[3], 16)
        if lineBits & filterBit == filterBit:
            output.append(line)
    return '\r\n'.join(output) + '\r\n'

def safeloadfi(dlc_path, name, mode='rb'):
    """safeloadfi : string -> string

    Safely load contents of a file, given a filename,
    and closing the file afterward.
    """
    try:
        with open(os.path.join(dlc_path, name), mode) as f:
            return f.read()
    except:
        return None

def download_count(dlc_path, post):
    """Handle download count request."""
    if post["gamecd"] in gamecodes_return_random_file:
        return "1"
    elif os.path.exists(dlc_path):
        if os.path.isfile(os.path.join(dlc_path, "_list.txt")):
            attr1 = post.get("attr1", None)
            attr2 = post.get("attr2", None)
            attr3 = post.get("attr3", None)

            dlc_file = safeloadfi(dlc_path, "_list.txt")
            ls = filter_list(dlc_file, attr1, attr2, attr3)
            count = get_file_count(ls)
        else:
            count = len(os.listdir(dlc_path))
        return "%d" % count
    else:
        return "0"

def download_size(dlc_path, name):
    """Return download filename and size.

    Used in download list.
    """
    return (name, str(os.path.getsize(os.path.join(dlc_path, name))))

def download_list(dlc_path, post):
    """Handle download list request.

    Look for a list file first. If the list file exists, send the
    entire thing back to the client.
    """
    # Get list file
    if not os.path.exists(dlc_path):
        return "\r\n"
    elif os.path.isfile(os.path.join(dlc_path, "_list.txt")):
        list_data = safeloadfi(dlc_path, "_list.txt") or "\r\n"
    else:
        # Doesn't have _list.txt file
        try:
            ls = [
                download_size(dlc_path, name)
                for name in sorted(os.listdir(dlc_path))
            ]
            list_data = "\r\n".join("\t\t\t\t\t".join(f) for f in ls) + "\r\n"
        except:
            return "\r\n"

    attr1 = post.get("attr1", None)
    attr2 = post.get("attr2", None)
    attr3 = post.get("attr3", None)

    if post["gamecd"].startswith("IRA") and attr1.startswith("MYSTERY"):
        # Pokemon BW Mystery Gifts, until we have a better solution for that
        ret = filter_list(list_data, attr1, attr2, attr3)
        ret = filter_list_g5_mystery_gift(ret, post["rhgamecd"])
        return filter_list_by_date(ret, post["token"])
    elif post["gamecd"] in gamecodes_return_random_file:
        # Pokemon Gen 4 Mystery Gifts, same here
        ret = filter_list(list_data, attr1, attr2, attr3)
        return filter_list_by_date(ret, post["token"])
    else:
        # Default case for most games
        num = post.get("num", None)
        if num is not None:
            num = int(num)

        offset = post.get("offset", None)
        if offset is not None:
            offset = int(offset)

        return filter_list(list_data, attr1, attr2, attr3, num, offset)

def download_contents(dlc_path, post):
    """Handle download contents request.

    Get only the base filename just in case there is a path involved
    somewhere in the filename string.
    """
    contents = os.path.basename(post["contents"])
    return safeloadfi(dlc_path, contents)
<commit_msg>DLS1: Fix filters on count action w/o list files<commit_after>"""DWC Network Server Emulator

    Copyright (C) 2014 polaris-
    Copyright (C) 2014 ToadKing
    Copyright (C) 2016 Sepalani

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import os
import random
import time

from gamespy.gs_database import GamespyDatabase

# If a game from this list requests a file listing, the server will return
# that only one exists and return a random one.
# This is used for Mystery Gift distribution on Generation 4 Pokemon games
gamecodes_return_random_file = [
    'ADAD',
    'ADAE',
    'ADAF',
    'ADAI',
    'ADAJ',
    'ADAK',
    'ADAS',
    'CPUD',
    'CPUE',
    'CPUF',
    'CPUI',
    'CPUJ',
    'CPUK',
    'CPUS',
    'IPGD',
    'IPGE',
    'IPGF',
    'IPGI',
    'IPGJ',
    'IPGK',
    'IPGS'
]

def get_file_count(data):
    return sum(1 for line in data.splitlines() if line)

def filter_list(data, attr1=None, attr2=None, attr3=None,
                num=None, offset=None):
    """Filter the list based on the attribute fields.

    If nothing matches, at least return a newline.
    Pokemon BW at least expects this and will error without it.
    """
    if attr1 is None and attr2 is None and attr3 is None and \
       num is None and offset is None:
        # Nothing to filter, just return the input data
        return data

    def attrs(data):
        """Filter attrs."""
        def nc(a, b):
            """Filter nc."""
            return a is None or a == b
        return \
            len(data) == 6 and \
            nc(attr1, data[2]) and \
            nc(attr2, data[3]) and \
            nc(attr3, data[4])
    output = filter(lambda line: attrs(line.split("\t")), data.splitlines())

    if offset is not None:
        output = output[offset:]

    if num is not None:
        output = output[:num]

    return '\r\n'.join(output) + '\r\n'

def filter_list_random_files(data, count):
    """Get [count] random files from the filelist."""
    samples = random.sample(data.splitlines(), count)
    return '\r\n'.join(samples) + '\r\n'

def filter_list_by_date(data, token):
    """Allow user to control which file to receive by setting
    the local date selected file will be the one at
    index (day of year) mod (file count)."""
    try:
        userData = GamespyDatabase().get_nas_login(token)
        date = time.strptime(userData['devtime'], '%y%m%d%H%M%S')
        files = data.splitlines()
        ret = files[(int(date.tm_yday) - 1) % len(files)] + '\r\n'
    except:
        ret = filter_list_random_files(data, 1)
    return ret

def filter_list_g5_mystery_gift(data, rhgamecd):
    """Custom selection for generation 5 mystery gifts, so that the random
    or data-based selection still works properly."""
    if rhgamecd[2] == 'A':
        filterBit = 0x100000
    elif rhgamecd[2] == 'B':
        filterBit = 0x200000
    elif rhgamecd[2] == 'D':
        filterBit = 0x400000
    elif rhgamecd[2] == 'E':
        filterBit = 0x800000
    else:
        # unknown game, can't filter
        return data

    output = []
    for line in data.splitlines():
        lineBits = int(line.split('\t')[3], 16)
        if lineBits & filterBit == filterBit:
            output.append(line)
    return '\r\n'.join(output) + '\r\n'

def safeloadfi(dlc_path, name, mode='rb'):
    """safeloadfi : string -> string

    Safely load contents of a file, given a filename,
    and closing the file afterward.
    """
    try:
        with open(os.path.join(dlc_path, name), mode) as f:
            return f.read()
    except:
        return None

def download_count(dlc_path, post):
    """Handle download count request."""
    if post["gamecd"] in gamecodes_return_random_file:
        return "1"
    if os.path.exists(dlc_path):
        attr1 = post.get("attr1", None)
        attr2 = post.get("attr2", None)
        attr3 = post.get("attr3", None)
        if os.path.isfile(os.path.join(dlc_path, "_list.txt")):
            dlc_file = safeloadfi(dlc_path, "_list.txt")
            ls = filter_list(dlc_file, attr1, attr2, attr3)
            return "{}".format(get_file_count(ls))
        elif attr1 is None and attr2 is None and attr3 is None:
            return "{}".format(len(os.listdir(dlc_path)))
    return "0"

def download_size(dlc_path, name):
    """Return download filename and size.

    Used in download list.
    """
    return (name, str(os.path.getsize(os.path.join(dlc_path, name))))

def download_list(dlc_path, post):
    """Handle download list request.

    Look for a list file first. If the list file exists, send the
    entire thing back to the client.
    """
    # Get list file
    if not os.path.exists(dlc_path):
        return "\r\n"
    elif os.path.isfile(os.path.join(dlc_path, "_list.txt")):
        list_data = safeloadfi(dlc_path, "_list.txt") or "\r\n"
    else:
        # Doesn't have _list.txt file
        try:
            ls = [
                download_size(dlc_path, name)
                for name in sorted(os.listdir(dlc_path))
            ]
            list_data = "\r\n".join("\t\t\t\t\t".join(f) for f in ls) + "\r\n"
        except:
            return "\r\n"

    attr1 = post.get("attr1", None)
    attr2 = post.get("attr2", None)
    attr3 = post.get("attr3", None)

    if post["gamecd"].startswith("IRA") and attr1.startswith("MYSTERY"):
        # Pokemon BW Mystery Gifts, until we have a better solution for that
        ret = filter_list(list_data, attr1, attr2, attr3)
        ret = filter_list_g5_mystery_gift(ret, post["rhgamecd"])
        return filter_list_by_date(ret, post["token"])
    elif post["gamecd"] in gamecodes_return_random_file:
        # Pokemon Gen 4 Mystery Gifts, same here
        ret = filter_list(list_data, attr1, attr2, attr3)
        return filter_list_by_date(ret, post["token"])
    else:
        # Default case for most games
        num = post.get("num", None)
        if num is not None:
            num = int(num)

        offset = post.get("offset", None)
        if offset is not None:
            offset = int(offset)

        return filter_list(list_data, attr1, attr2, attr3, num, offset)

def download_contents(dlc_path, post):
    """Handle download contents request.

    Get only the base filename just in case there is a path involved
    somewhere in the filename string.
    """
    contents = os.path.basename(post["contents"])
    return safeloadfi(dlc_path, contents)
<|endoftext|>


<commit_before># Copyright (C) 2010,2011  Chris Lalancette <clalance@redhat.com>
# Copyright (C) 2012-2015  Chris Lalancette <clalancette@gmail.com>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Fedora installation
"""

import os

import oz.ozutil
import oz.RedHat
import oz.OzException

class FedoraGuest(oz.RedHat.RedHatLinuxCDYumGuest):
    """
    Class for Fedora 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, and 23 installation.
    """
    def __init__(self, tdl, config, auto, nicmodel, haverepo, diskbus,
                 brokenisomethod, output_disk=None, macaddress=None,
                 assumed_update=None):
        directkernel = "cpio"
        if tdl.update in ["16", "17"]:
            directkernel = None
        self.assumed_update = assumed_update

        oz.RedHat.RedHatLinuxCDYumGuest.__init__(self, tdl, config, auto,
                                                 output_disk, nicmodel, diskbus,
                                                 True, True, directkernel,
                                                 macaddress)

        if self.assumed_update is not None:
            self.log.warning("==== WARN: TDL contains Fedora update %s, which is newer than Oz knows about; pretending this is Fedora %s, but this may fail ====", tdl.update, assumed_update)

        self.haverepo = haverepo
        self.brokenisomethod = brokenisomethod

    def _modify_iso(self):
        """
        Method to modify the ISO for autoinstallation.
        """
        self._copy_kickstart(os.path.join(self.iso_contents, "ks.cfg"))

        if self.tdl.update in ["17", "18", "19", "20", "21", "22", "23"]:
            initrdline = "  append initrd=initrd.img ks=cdrom:/dev/cdrom:/ks.cfg"
        else:
            initrdline = "  append initrd=initrd.img ks=cdrom:/ks.cfg"
        if self.tdl.installtype == "url":
            if self.haverepo:
                initrdline += " repo="
            else:
                initrdline += " method="
            initrdline += self.url + "\n"
        else:
            # if the installtype is iso, then due to a bug in anaconda we leave
            # out the method completely
            if not self.brokenisomethod:
                initrdline += " method=cdrom:/dev/cdrom"
            initrdline += "\n"
        self._modify_isolinux(initrdline)

    def generate_diskimage(self, size=10, force=False):
        """
        Method to generate a diskimage.  By default, a blank diskimage of
        10GB will be created; the caller can override this with the size
        parameter, specified in GB.  If force is False (the default), then
        a diskimage will not be created if a cached JEOS is found.  If
        force is True, a diskimage will be created regardless of whether a
        cached JEOS exists.  See the oz-install man page for more
        information about JEOS caching.
        """
        createpart = False
        if self.tdl.update in ["11", "12"]:
            # If given a blank diskimage, Fedora 11/12 stops very early in
            # install with a message about losing all of your data on the
            # drive (it differs between them).
            #
            # To avoid that message, just create a partition table that spans
            # the entire disk
            createpart = True
        return self._internal_generate_diskimage(size, force, createpart)

    def get_auto_path(self):
        """
        Method to create the correct path to the Fedora kickstart files.
        """
        # If we are doing our best with an unknown Fedora update, use the
        # newest known auto file; otherwise, do the usual thing.
        if self.assumed_update is not None:
            return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.assumed_update + ".auto")
        else:
            return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.tdl.update + ".auto")

def get_class(tdl, config, auto, output_disk=None, netdev=None, diskbus=None,
              macaddress=None):
    """
    Factory method for Fedora installs.
    """
    newer_distros = ["21", "22", "23"]

    if int(tdl.update) > int(newer_distros[-1]):
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, False,
                           output_disk, macaddress, newer_distros[-1])

    if tdl.update in newer_distros:
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, False,
                           output_disk, macaddress, None)

    if tdl.update in ["10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20"]:
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, True,
                           output_disk, macaddress, None)

    if tdl.update in ["7", "8", "9"]:
        return FedoraGuest(tdl, config, auto, netdev, False, diskbus, False,
                           output_disk, macaddress, None)

def get_supported_string():
    """
    Return supported versions as a string.
    """
    return "Fedora: 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23"
<commit_msg>Support 'rawhide' as Fedora version<commit_after># Copyright (C) 2010,2011  Chris Lalancette <clalance@redhat.com>
# Copyright (C) 2012-2015  Chris Lalancette <clalancette@gmail.com>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Fedora installation
"""

import os

import oz.ozutil
import oz.RedHat
import oz.OzException

class FedoraGuest(oz.RedHat.RedHatLinuxCDYumGuest):
    """
    Class for Fedora 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, and 23 installation.
    """
    def __init__(self, tdl, config, auto, nicmodel, haverepo, diskbus,
                 brokenisomethod, output_disk=None, macaddress=None,
                 assumed_update=None):
        directkernel = "cpio"
        if tdl.update in ["16", "17"]:
            directkernel = None
        self.assumed_update = assumed_update

        oz.RedHat.RedHatLinuxCDYumGuest.__init__(self, tdl, config, auto,
                                                 output_disk, nicmodel, diskbus,
                                                 True, True, directkernel,
                                                 macaddress)

        if self.assumed_update is not None:
            self.log.warning("==== WARN: TDL contains Fedora update %s, which is newer than Oz knows about; pretending this is Fedora %s, but this may fail ====", tdl.update, assumed_update)

        self.haverepo = haverepo
        self.brokenisomethod = brokenisomethod

    def _modify_iso(self):
        """
        Method to modify the ISO for autoinstallation.
        """
        self._copy_kickstart(os.path.join(self.iso_contents, "ks.cfg"))

        if self.tdl.update in ["17", "18", "19", "20", "21", "22", "23"]:
            initrdline = "  append initrd=initrd.img ks=cdrom:/dev/cdrom:/ks.cfg"
        else:
            initrdline = "  append initrd=initrd.img ks=cdrom:/ks.cfg"
        if self.tdl.installtype == "url":
            if self.haverepo:
                initrdline += " repo="
            else:
                initrdline += " method="
            initrdline += self.url + "\n"
        else:
            # if the installtype is iso, then due to a bug in anaconda we leave
            # out the method completely
            if not self.brokenisomethod:
                initrdline += " method=cdrom:/dev/cdrom"
            initrdline += "\n"
        self._modify_isolinux(initrdline)

    def generate_diskimage(self, size=10, force=False):
        """
        Method to generate a diskimage.  By default, a blank diskimage of
        10GB will be created; the caller can override this with the size
        parameter, specified in GB.  If force is False (the default), then
        a diskimage will not be created if a cached JEOS is found.  If
        force is True, a diskimage will be created regardless of whether a
        cached JEOS exists.  See the oz-install man page for more
        information about JEOS caching.
        """
        createpart = False
        if self.tdl.update in ["11", "12"]:
            # If given a blank diskimage, Fedora 11/12 stops very early in
            # install with a message about losing all of your data on the
            # drive (it differs between them).
            #
            # To avoid that message, just create a partition table that spans
            # the entire disk
            createpart = True
        return self._internal_generate_diskimage(size, force, createpart)

    def get_auto_path(self):
        """
        Method to create the correct path to the Fedora kickstart files.
        """
        # If we are doing our best with an unknown Fedora update, use the
        # newest known auto file; otherwise, do the usual thing.
        if self.assumed_update is not None:
            return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.assumed_update + ".auto")
        else:
            return oz.ozutil.generate_full_auto_path(self.tdl.distro + self.tdl.update + ".auto")

def get_class(tdl, config, auto, output_disk=None, netdev=None, diskbus=None,
              macaddress=None):
    """
    Factory method for Fedora installs.
    """
    newer_distros = ["21", "22", "23"]

    if tdl.update == 'rawhide' or int(tdl.update) > int(newer_distros[-1]):
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, False,
                           output_disk, macaddress, newer_distros[-1])

    if tdl.update in newer_distros:
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, False,
                           output_disk, macaddress, None)

    if tdl.update in ["10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20"]:
        if netdev is None:
            netdev = 'virtio'
        if diskbus is None:
            diskbus = 'virtio'
        return FedoraGuest(tdl, config, auto, netdev, True, diskbus, True,
                           output_disk, macaddress, None)

    if tdl.update in ["7", "8", "9"]:
        return FedoraGuest(tdl, config, auto, netdev, False, diskbus, False,
                           output_disk, macaddress, None)

def get_supported_string():
    """
    Return supported versions as a string.
    """
    return "Fedora: 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23"
<|endoftext|>