<commit_before># -*- coding: utf8 -*-
import networkx
import numpy
from csv import DictReader, DictWriter
from matplotlib import pyplot
from matplotlib.patches import FancyArrowPatch
from os.path import basename, isdir, join, normpath
from re import finditer, match, MULTILINE
from subprocess import Popen, PIPE

from core.utils.rpla import get_available_platforms, get_motes_from_simulation

__all__ = [
    'parsing_chain',
]

# *************************************** MAIN PARSING FUNCTION ****************************************
def parsing_chain(path):
    """
    This function chains the conversion and drawings for exploiting the data collected while running a Cooja
     simulation into relevant results.

    :param path: path to the experiment (including [with-|without-malicious])
    """
    assert isdir(path)
    __convert_pcap_to_csv(path)
    __convert_powertracker_log_to_csv(path)
    __draw_dodag(path)
    __draw_power_barchart(path)

# *********************************** SIMULATION PARSING FUNCTIONS *************************************
def __convert_pcap_to_csv(path):
    """
    This function creates a CSV file (to ./results) from a PCAP file (from ./data).
    This is inspired from https://github.com/sieben/makesense/blob/master/makesense/parser.py.

    :param path: path to the experiment (including [with-|without-malicious])
    """
    data, results = join(path, 'data'), join(path, 'results')
    try:
        p = Popen(['tshark',
                   '-T', 'fields',
                   '-E', 'header=y',
                   '-E', 'separator=,',
                   '-e', 'frame.time',
                   '-e', 'frame.len',
                   '-e', 'wpan.src64',
                   '-e', 'wpan.dst64',
                   '-e', 'icmpv6.type',
                   '-e', 'ipv6.src',
                   '-e', 'ipv6.dst',
                   '-e', 'icmpv6.code',
                   '-e', 'data.data',
                   '-r', join(data, 'output.pcap')], stdout=PIPE)
        out, _ = p.communicate()
    except OSError:
        out = b"[ERROR] Tshark is not installed !"
    with open(join(results, 'pcap.csv'), 'wb') as f:
        f.write(out)

PT_ITEMS = ['monitored', 'on', 'tx', 'rx', 'int']
PT_REGEX = r'^({})_(?P<mote_id>\d+) {} (?P<{}>\d+)'

def __convert_powertracker_log_to_csv(path):
    """
    This function creates a CSV file (to ./results) from a PowerTracker log file (from ./data).
    This is inspired from https://github.com/sieben/makesense/blob/master/makesense/parser.py.

    :param path: path to the experiment (including [with-|without-malicious])
    """
    platforms = [p.capitalize() for p in get_available_platforms()]
    data, results = join(path, 'data'), join(path, 'results')
    with open(join(data, 'powertracker.log')) as f:
        log = f.read()
    iterables, fields = [], ['mote_id']
    for it in PT_ITEMS:
        time_field = '{}_time'.format(it)
        iterables.append(finditer(PT_REGEX.format('|'.join(platforms), it.upper(), time_field), log, MULTILINE))
        fields.append(time_field)
    with open(join(results, 'powertracker.csv'), 'w') as f:
        writer = DictWriter(f, delimiter=',', fieldnames=fields)
        writer.writeheader()
        for matches in zip(*iterables):
            row = {}
            for m in matches:
                row.update((k, int(v)) for k, v in m.groupdict().items())
            for it in PT_ITEMS:
                time_field = '{}_time'.format(it)
                row[time_field] = float(row[time_field] / 10 ** 6)
            writer.writerow(row)

RELATIONSHIP_REGEX = r'^\d+\s+ID\:(?P<mote_id>\d+)\s+#L\s+(?P<parent_id>\d+)\s+(?P<flag>\d+)$'

def __draw_dodag(path):
    """
    This function draws the DODAG (to ./results) from the list of motes (from ./simulation.csc) and the list of
     edges (from ./data/relationships.log).

    :param path: path to the experiment (including [with-|without-malicious])
    """
    pyplot.clf()
    with_malicious = (basename(normpath(path)) == 'with-malicious')
    data, results = join(path, 'data'), join(path, 'results')
    with open(join(data, 'relationships.log')) as f:
        relationships = f.read()
    # first, check if the mote relationships were recorded
    if len(relationships.strip()) == 0:
        return
    # retrieve motes and their colors
    dodag = networkx.DiGraph()
    motes = get_motes_from_simulation(join(path, 'simulation.csc'))
    dodag.add_nodes_from(motes.keys())
    colors = []
    for n, p in motes.items():
        x, y = p
        dodag.node[n]['pos'] = motes[n] = (x, -y)
        colors.append('green' if n == 0 else ('yellow' if not with_malicious or
                                              (with_malicious and 0 < n < len(motes) - 1) else 'red'))
    # retrieve edges from relationships.log
    edges = {}
    for relationship in relationships.split('\n'):
        try:
            d = match(RELATIONSHIP_REGEX, relationship.strip()).groupdict()
            if int(d['flag']) == 0:
                continue
            mote, parent = int(d['mote_id']), int(d['parent_id'])
            edges[mote] = parent
        except AttributeError:
            continue
    # now, fill in the graph with edges
    dodag.add_edges_from(edges.items())
    # finally, draw the graph
    networkx.draw(dodag, motes, node_color=colors, with_labels=True)
    pyplot.savefig(join(results, 'dodag.png'), arrow_style=FancyArrowPatch)

def __draw_power_barchart(path):
    """
    This function plots the average power tracking data from the CSV at:
     [EXPERIMENT]/[with-|without-malicious]/results/powertracker.csv

    :param path: path to the experiment (including [with-|without-malicious])
    :return:
    """
    pyplot.clf()
    items = ['on', 'tx', 'rx', 'int']
    series = {i: [] for i in items}
    averages, c = {}, 0
    with open(join(path, 'results', 'powertracker.csv')) as f:
        for row in DictReader(f):
            mid = int(row['mote_id'])
            averages.setdefault(mid, {i: 0.0 for i in items})
            for s, k in zip(items, [i + '_time' for i in items]):
                averages[mid][s] += float(row[k])
            c += 1
    n = len(averages)
    ind = numpy.arange(n)
    c //= n
    averages = {mid: {k: v / c for k, v in avg.items()} for mid, avg in averages.items()}
    for mid, avg in sorted(averages.items(), key=lambda x: x[0]):
        for k, v in series.items():
            v.append(avg[k])
    width = 0.5
    plots = []
    for s, color in zip(items, ['r', 'b', 'g', 'y']):
        plots.append(pyplot.bar(ind, series[s], width, color=color))
    pyplot.title("Power tracking per mote")
    pyplot.xticks(ind + width / 2., tuple(sorted(averages.keys())))
    pyplot.yticks(numpy.arange(0, 31, 10))
    pyplot.ylabel("Consumed power (%)")
    pyplot.legend(tuple(p[0] for p in plots), tuple(i.upper() for i in items))
    pyplot.savefig(join(path, 'results', 'powertracking.png'))
<commit_msg>Minor improvements to simulation results parser<commit_after># -*- coding: utf8 -*-
import networkx
import numpy
from csv import DictReader, DictWriter
from matplotlib import pyplot
from matplotlib.patches import FancyArrowPatch
from os.path import basename, isdir, join, normpath
from re import finditer, match, MULTILINE
from subprocess import Popen, PIPE

from core.utils.rpla import get_available_platforms, get_motes_from_simulation

__all__ = [
    'parsing_chain',
]

# *************************************** MAIN PARSING FUNCTION ****************************************
def parsing_chain(path, logger=None):
    """
    This function chains the conversion and drawings for exploiting the data collected while running a Cooja
     simulation into relevant results.

    :param path: path to the experiment (including [with-|without-malicious])
    :param logger: logging.Logger instance
    """
    assert isdir(path)
    __convert_pcap_to_csv(path, logger)
    __convert_powertracker_log_to_csv(path, logger)
    __draw_dodag(path, logger)
    __draw_power_barchart(path, logger)

# *********************************** SIMULATION PARSING FUNCTIONS *************************************
def __convert_pcap_to_csv(path, logger=None):
    """
    This function creates a CSV file (to ./results) from a PCAP file (from ./data).
    This is inspired from https://github.com/sieben/makesense/blob/master/makesense/parser.py.

    :param path: path to the experiment (including [with-|without-malicious])
    :param logger: logging.Logger instance
    """
    if logger:
        logger.debug(" > Converting PCAP to CSV...")
    data, results = join(path, 'data'), join(path, 'results')
    try:
        p = Popen(['tshark',
                   '-T', 'fields',
                   '-E', 'header=y',
                   '-E', 'separator=,',
                   '-e', 'frame.time',
                   '-e', 'frame.len',
                   '-e', 'wpan.src64',
                   '-e', 'wpan.dst64',
                   '-e', 'icmpv6.type',
                   '-e', 'ipv6.src',
                   '-e', 'ipv6.dst',
                   '-e', 'icmpv6.code',
                   '-e', 'data.data',
                   '-r', join(data, 'output.pcap')], stdout=PIPE)
        out, _ = p.communicate()
    except OSError:
        out = b"Tshark is not installed !"
        if logger:
            logger.warn(out)
    with open(join(results, 'pcap.csv'), 'wb') as f:
        f.write(out)

PT_ITEMS = ['monitored', 'on', 'tx', 'rx', 'int']
PT_REGEX = r'^({})_(?P<mote_id>\d+) {} (?P<{}>\d+)'

def __convert_powertracker_log_to_csv(path, logger=None):
    """
    This function creates a CSV file (to ./results) from a PowerTracker log file (from ./data).
    This is inspired from https://github.com/sieben/makesense/blob/master/makesense/parser.py.

    :param path: path to the experiment (including [with-|without-malicious])
    :param logger: logging.Logger instance
    """
    if logger:
        logger.debug(" > Converting power tracking log to CSV...")
    platforms = [p.capitalize() for p in get_available_platforms()]
    data, results = join(path, 'data'), join(path, 'results')
    with open(join(data, 'powertracker.log')) as f:
        log = f.read()
    iterables, fields = [], ['mote_id']
    for it in PT_ITEMS:
        time_field = '{}_time'.format(it)
        iterables.append(finditer(PT_REGEX.format('|'.join(platforms), it.upper(), time_field), log, MULTILINE))
        fields.append(time_field)
    with open(join(results, 'powertracker.csv'), 'w') as f:
        writer = DictWriter(f, delimiter=',', fieldnames=fields)
        writer.writeheader()
        for matches in zip(*iterables):
            row = {}
            for m in matches:
                row.update((k, int(v)) for k, v in m.groupdict().items())
            for it in PT_ITEMS:
                time_field = '{}_time'.format(it)
                row[time_field] = float(row[time_field] / 10 ** 6)
            writer.writerow(row)

RELATIONSHIP_REGEX = r'^\d+\s+ID\:(?P<mote_id>\d+)\s+#L\s+(?P<parent_id>\d+)\s+(?P<flag>\d+)$'

def __draw_dodag(path, logger=None):
    """
    This function draws the DODAG (to ./results) from the list of motes (from ./simulation.csc) and the list of
     edges (from ./data/relationships.log).

    :param path: path to the experiment (including [with-|without-malicious])
    :param logger: logging.Logger instance
    """
    if logger:
        logger.debug(" > Drawing DODAG to PNG...")
    pyplot.clf()
    with_malicious = (basename(normpath(path)) == 'with-malicious')
    data, results = join(path, 'data'), join(path, 'results')
    with open(join(data, 'relationships.log')) as f:
        relationships = f.read()
    # first, check if the mote relationships were recorded
    if len(relationships.strip()) == 0:
        if logger:
            logger.warn("Relationships log file is empty !")
        return
    # retrieve motes and their colors
    dodag = networkx.DiGraph()
    motes = get_motes_from_simulation(join(path, 'simulation.csc'))
    dodag.add_nodes_from(motes.keys())
    colors = []
    for n, p in motes.items():
        x, y = p
        dodag.node[n]['pos'] = motes[n] = (x, -y)
        colors.append('green' if n == 0 else ('yellow' if not with_malicious or
                                              (with_malicious and 0 < n < len(motes) - 1) else 'red'))
    # retrieve edges from relationships.log
    edges = {}
    for relationship in relationships.split('\n'):
        try:
            d = match(RELATIONSHIP_REGEX, relationship.strip()).groupdict()
            if int(d['flag']) == 0:
                continue
            mote, parent = int(d['mote_id']), int(d['parent_id'])
            edges[mote] = parent
        except AttributeError:
            continue
    # now, fill in the graph with edges
    dodag.add_edges_from(edges.items())
    # finally, draw the graph
    networkx.draw(dodag, motes, node_color=colors, with_labels=True)
    pyplot.savefig(join(results, 'dodag.png'), arrow_style=FancyArrowPatch)

def __draw_power_barchart(path, logger=None):
    """
    This function plots the average power tracking data from the CSV at:
     [EXPERIMENT]/[with-|without-malicious]/results/powertracker.csv

    :param path: path to the experiment (including [with-|without-malicious])
    :param logger: logging.Logger instance
    """
    if logger:
        logger.debug(" > Drawing power bar chart to PNG...")
    pyplot.clf()
    items = ['on', 'tx', 'rx', 'int']
    series = {i: [] for i in items}
    averages, c = {}, 0
    with open(join(path, 'results', 'powertracker.csv')) as f:
        for row in DictReader(f):
            mid = int(row['mote_id'])
            averages.setdefault(mid, {i: 0.0 for i in items})
            for s, k in zip(items, [i + '_time' for i in items]):
                averages[mid][s] += float(row[k])
            c += 1
    n = len(averages)
    ind = numpy.arange(n)
    c //= n
    averages = {mid: {k: v / c for k, v in avg.items()} for mid, avg in averages.items()}
    for mid, avg in sorted(averages.items(), key=lambda x: x[0]):
        for k, v in series.items():
            v.append(avg[k])
    width = 0.5
    plots = []
    for s, color in zip(items, ['r', 'b', 'g', 'y']):
        plots.append(pyplot.bar(ind, series[s], width, color=color))
    pyplot.title("Power tracking per mote")
    pyplot.xticks(ind + width / 2., tuple(sorted(averages.keys())))
    pyplot.yticks(numpy.arange(0, 31, 10))
    pyplot.ylabel("Consumed power (%)")
    pyplot.legend(tuple(p[0] for p in plots), tuple(i.upper() for i in items))
    pyplot.savefig(join(path, 'results', 'powertracking.png'))
<|endoftext|>


<commit_before>from itertools import product, combinations_with_replacement
from copy import deepcopy
from hoomd.util import to_camel_case, is_iterable
from hoomd.typeconverter import TypeConverter, TypeConversionError
from hoomd.typeconverter import from_type_converter_input_to_default

# Psudonym for None that states an argument is required to be supplied by the
# user
RequiredArg = None

def has_str_elems(obj):
    '''Returns True if all elements of iterable are str.'''
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    '''Returns True if object is iterable with respect to types.'''
    return is_iterable(obj) and has_str_elems(obj)

def proper_type_return(val):
    '''Expects and requires a dictionary with type keys.'''
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        if 'explicit_defaults' in kwargs.keys():
            explicit_defaults = kwargs['explicit_defaults']
            del kwargs['explicit_defaults']
        else:
            explicit_defaults = None

        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")
        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            default_arg = kwargs
        else:
            default_arg = args[0]
        self._default = from_type_converter_input_to_default(default_arg,
                                                             explicit_defaults)
        self._type_converter = TypeConverter.from_default(default_arg)

    def _validate_values(self, val):
        try:
            val = self._type_converter(val)
        except TypeConversionError as err:
            if len(err.args) > 1:
                raise TypeConversionError("TypeParameter {} key {} has "
                                          "conversion error {}".format(
                                              self, err.args[1], err.args[0]))
            else:
                raise TypeConversionError("TypeParameter {} has "
                                          "conversion error {}".format(
                                              self, err.args[0]))
        if type(self._default) == dict:
            dft_copy = self.default
            dft_keys = set(dft_copy.keys())
            if len(dft_keys.intersection(val.keys())) != len(val.keys()):
                raise ValueError("Keys must be a subset of available keys ")
            dft_copy.update(val)
            val = dft_copy
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        '''Validate key given regardless of key length.'''
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        '''Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        '''
        if isinstance(key, str):
            return [key]
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        '''Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        '''
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([not is_good_iterable(v) and not isinstance(v, str)
                    for v in key]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        '''Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        '''
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    @property
    def default(self):
        return deepcopy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._type_converter(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
            self._default.update(new_default)
        else:
            self._default = new_default

class TypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            try:
                vals[key] = self._dict[key]
            except KeyError:
                vals[key] = self.default
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            self._dict[key] = val

    def keys(self):
        if self._len_keys == 1:
            yield from self._dict.keys()
        else:
            for key in self._dict.keys():
                yield tuple(sorted(list(key)))

    def to_dict(self):
        return self._dict

class AttachedTypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, cpp_obj, param_name,
                 type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict.default
        self._type_converter = type_param_dict._type_converter
        # add all types to c++
        for key in self.keys():
            try:
                self[key] = type_param_dict[key]
            except ValueError as verr:
                raise ValueError("Type {} ".format(key) + verr.args[0])

    def to_dettached(self):
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        type_param_dict._type_converter = self._type_converter
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        '''Includes key check for existing simulation keys.'''
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    def _validate_values(self, val):
        val = super()._validate_values(val)
        if isinstance(val, dict):
            not_set_keys = []
            for k, v in val.items():
                if v is None:
                    not_set_keys.append(k)
            if not_set_keys != []:
                raise ValueError("{} were not set.".format(not_set_keys))
        return val

    @property
    def _setter(self):
        return 'set' + to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + to_camel_case(self._param_name)

    def keys(self):
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 1:
            yield from single_keys
        else:
            for key in combinations_with_replacement(single_keys,
                                                     self._len_keys):
                yield tuple(sorted(list(key)))

    def to_dict(self):
        rtn_dict = dict()
        for key in self.keys():
            rtn_dict[key] = getattr(self._cpp_obj, self._getter)(key)
        return rtn_dict
<commit_msg>Add ParameterDict class<commit_after>from itertools import product, combinations_with_replacement
from copy import deepcopy
from hoomd.util import to_camel_case, is_iterable
from hoomd.typeconverter import TypeConverter, TypeConversionError
from hoomd.typeconverter import from_type_converter_input_to_default

# Psudonym for None that states an argument is required to be supplied by the
# user
RequiredArg = None

def has_str_elems(obj):
    '''Returns True if all elements of iterable are str.'''
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    '''Returns True if object is iterable with respect to types.'''
    return is_iterable(obj) and has_str_elems(obj)

def proper_type_return(val):
    '''Expects and requires a dictionary with type keys.'''
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        if 'explicit_defaults' in kwargs.keys():
            explicit_defaults = kwargs['explicit_defaults']
            del kwargs['explicit_defaults']
        else:
            explicit_defaults = None

        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")
        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            default_arg = kwargs
        else:
            default_arg = args[0]
        self._default = from_type_converter_input_to_default(default_arg,
                                                             explicit_defaults)
        self._type_converter = TypeConverter.from_default(default_arg)

    def _validate_values(self, val):
        try:
            val = self._type_converter(val)
        except TypeConversionError as err:
            if len(err.args) > 1:
                raise TypeConversionError("TypeParameter {} key {} has "
                                          "conversion error {}".format(
                                              self, err.args[1], err.args[0]))
            else:
                raise TypeConversionError("TypeParameter {} has "
                                          "conversion error {}".format(
                                              self, err.args[0]))
        if type(self._default) == dict:
            dft_copy = self.default
            dft_keys = set(dft_copy.keys())
            if len(dft_keys.intersection(val.keys())) != len(val.keys()):
                raise ValueError("Keys must be a subset of available keys ")
            dft_copy.update(val)
            val = dft_copy
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        '''Validate key given regardless of key length.'''
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        '''Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        '''
        if isinstance(key, str):
            return [key]
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        '''Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        '''
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([not is_good_iterable(v) and not isinstance(v, str)
                    for v in key]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        '''Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        '''
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    @property
    def default(self):
        return deepcopy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._type_converter(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
            self._default.update(new_default)
        else:
            self._default = new_default

class TypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            try:
                vals[key] = self._dict[key]
            except KeyError:
                vals[key] = self.default
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            self._dict[key] = val

    def keys(self):
        if self._len_keys == 1:
            yield from self._dict.keys()
        else:
            for key in self._dict.keys():
                yield tuple(sorted(list(key)))

    def to_dict(self):
        return self._dict

class AttachedTypeParameterDict(_ValidatedDefaultDict):

    def __init__(self, cpp_obj, param_name,
                 type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict.default
        self._type_converter = type_param_dict._type_converter
        # add all types to c++
        for key in self.keys():
            try:
                self[key] = type_param_dict[key]
            except ValueError as verr:
                raise ValueError("Type {} ".format(key) + verr.args[0])

    def to_dettached(self):
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        type_param_dict._type_converter = self._type_converter
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        val = self._validate_values(val)
        for key in self._yield_keys(key):
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        '''Includes key check for existing simulation keys.'''
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    def _validate_values(self, val):
        val = super()._validate_values(val)
        if isinstance(val, dict):
            not_set_keys = []
            for k, v in val.items():
                if v is None:
                    not_set_keys.append(k)
            if not_set_keys != []:
                raise ValueError("{} were not set.".format(not_set_keys))
        return val

    @property
    def _setter(self):
        return 'set' + to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + to_camel_case(self._param_name)

    def keys(self):
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 1:
            yield from single_keys
        else:
            for key in combinations_with_replacement(single_keys,
                                                     self._len_keys):
                yield tuple(sorted(list(key)))

    def to_dict(self):
        rtn_dict = dict()
        for key in self.keys():
            rtn_dict[key] = getattr(self._cpp_obj, self._getter)(key)
        return rtn_dict

class ParameterDict(dict):
    def __init__(self, explicit_defaults=None, **kwargs):
        self._type_converter = TypeConverter.from_default(kwargs)
        super().__init__(**from_type_converter_input_to_default(
            kwargs, explicit_defaults)
            )

    def __setitem__(self, key, value):
        if key not in self.keys():
            super().__setitem__(key, value)
            self._type_converter[key] = TypeConverter.from_default(value)
        else:
            super().__setitem__(key, self._type_converter[key](value))

    def update(self, dict_):
        for key, value in dict_.items():
            self[key] = value
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

class BadEntryException(Exception):
    """
    Thrown when entry data is invalid
    """
    pass

class Entry(object):
    """
    Represents a single entry in fathead output.txt datafile
    """
    def __init__(self, obj):
        # Type of entry. R = Redirect, A = Article.
        self.type = None

        # Unique key
        self.key = None

        # Alternative keys used for redirects
        self.alternative_keys = []
        self.parse(obj)

    def parse(self, input_obj):
        """
        Try to parse given input into a valid entry.
        Args:
            input_obj: TSV string or list of data.

        Returns:
            List of data
        Raises:
            Throws BadEntryException if data is invalid.
        """
        if isinstance(input_obj, str):
            processed = input_obj.split('\t')
            self.data = processed
        elif isinstance(input_obj, list):
            self.data = input_obj

        # Entry must contain 13 elements.
        if len(self.data) != 13:
            raise BadEntryException('Bad entry format')

        self.key = self.data[0]
        self.type = self.data[1]
        self.parse_alternative_keys()

        return self.data

    def parse_alternative_keys(self):
        """
        Find alternative keys to use in generated redirects

        Returns:
            Set of possible redirect entries
        """
        self.alternative_keys = set()

        if self.key.count('.') > 0:
            key_arr = self.key.split('.')
            method_name = key_arr[-1]
            package_name = key_arr[0]
            new_key = "{} {}".format(package_name, method_name)
            self.alternative_keys.add(new_key)
            self.alternative_keys.add(method_name)
        return self.alternative_keys

    def get_data(self):
        return self.data

    def get_key(self):
        return self.key

    def get_type(self):
        return self.type

    def get_redirects(self):
        # @TODO: Generate R entries
        return []

    def get_alternatives(self):
        return self.alternative_keys

def generate_redirects(f):
    # @TODO: work in progress
    possible_redirects = set()
    for line in f.readlines():
        try:
            # Parse entry
            entry = Entry(line)
            # Get all possible redirect entries
            possible_redirects = possible_redirects | entry.get_alternatives()
        except BadEntryException as e:
            pass  # Continue execution entry data is invalid.

if __name__ == "__main__":
    # Open output file for reading and writing.
    with open('output.txt', 'r+') as output_file:
        generate_redirects(output_file)
<commit_msg>Added redirect generation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

class BadEntryException(Exception):
    """
    Thrown when entry data is invalid
    """
    pass

class Entry(object):
    """
    Represents a single entry in fathead output.txt datafile
    """
    def __init__(self, obj):
        # Type of entry. R = Redirect, A = Article.
        self.entry_type = ''

        # Unique key
        self.key = ''

        # Referenced entry key (only for redirects)
        self.reference = ''
        self.abstract = ''
        self.anchor = ''

        # Alternative keys used for redirects
        self.alternative_keys = []
        self.parse(obj)

    def parse(self, input_obj):
        """
        Try to parse given input into a valid entry.
        Args:
            input_obj: TSV string or list of data.

        Returns:
            List of data
        Raises:
            Throws BadEntryException if data is invalid.
        """
        if isinstance(input_obj, str):
            processed = input_obj.split('\t')
            self.data = processed
        elif isinstance(input_obj, list):
            self.data = input_obj

        try:
            self.key = self.data[0].strip()
            self.entry_type = self.data[1].strip()
            self.reference = self.data[2].strip()
            if len(self.data) > 3:
                self.abstract = self.data[11].strip()
                self.anchor = self.data[12].strip()
            elif self.entry_type == 13 and len(self.data) != 13:
                raise BadEntryException
        except Exception as e:
            raise BadEntryException('Article had invalid number of elements.')

        if self.entry_type == 'A':
            self.parse_alternative_keys()

        return self.data

    def parse_alternative_keys(self):
        """
        Find alternative keys to use in generated redirects

        Returns:
            Set of possible redirect entries
        """
        self.alternative_keys = set()

        if '.' in self.key and self.entry_type == 'A':
            key_arr = self.key.split('.')
            method_name = key_arr[-1]
            package_name = key_arr[0]
            new_key = "{} {}".format(package_name, method_name)
            self.alternative_keys.add(new_key)
            self.alternative_keys.add(method_name)
        return self.alternative_keys

    def get_data(self):
        return self.data

    def get_key(self):
        return self.key

    def get_type(self):
        return self.entry_type

    def get_redirects(self):
        redirs = []

        for alt_key in list(self.alternative_keys):
            entry = Entry([
                alt_key,
                'R',
                self.key
            ])
            redirs.append(entry)

        return redirs

    def get_alternatives(self):
        return self.alternative_keys

    def get_entry(self):
        return '\t'.join([
            self.key,             # title / key
            self.entry_type,      # entry type
            self.reference,       # no redirect data
            '',                   # ignore
            '',                   # no categories
            '',                   # ignore
            '',                   # no related topics
            '',                   # ignore
            '',                   # add an external link back to page
            '',                   # no disambiguation
            '',                   # images
            self.abstract,        # abstract
            self.anchor           # anchor to specific section
        ])

    def __str__(self):
        return self.get_entry()

def generate_redirects(f):
    # @TODO: Cleanup
    # @TODO: Figure out why the duplicate count is ~20k
    output = dict()

    # For debugging purposes
    duplicate_count = 0

    for line in f.readlines():
        try:
            # Parse entry
            entry = Entry(line)
            key = entry.get_key()

            # Do we have the entry yet?
            if key not in output:
                output[key] = str(entry)
            else:
                duplicate_count += 1

            # Get all possible redirect entries
            redirects = entry.get_redirects()
            for redirect in redirects:
                key = redirect.get_key()
                if key not in output:
                    output[key] = str(entry)
                else:
                    duplicate_count += 1
        except BadEntryException as e:
            pass  # Continue execution entry data is invalid.

    print(duplicate_count)

    with open('output2.txt', 'w') as output_file:
        for key, line in output.items():
            tsv = '{}\n'.format(line)
            output_file.write(tsv)

if __name__ == "__main__":
    # Open output file for reading and writing.
    with open('output.txt', 'r+') as output_file:
        generate_redirects(output_file)
<|endoftext|>


<commit_before>"""
Example of using the state system. The Event system allows a player object to be
'trapped' in a special environment where different commands are available than normal.
This is very useful in order to implement anything from menus to npc-conversational
choices and inline text-editors.

This example uses the State system to create a simple menu.

To test out this example, add this module to the CUSTOM_COMMAND_MODULES tuple in
your game/settings.py as 'game.gamesrc.commands.examples.state_example' (see ./example.py
for another example). You need to restart the Evennia server before new files are
recognized.

Next enter the mud and give the command
> entermenu

Note that the help entries added to the state system with the auto_help flag are NOT
part of the normal help database, they are stored with the state and only accessible
from inside it (unless you also set the 'global_help' flag in the add_command(), in
which case it is also added to the global help system). If you want to describe the
state itself in more detail you should add that to the main help index manually.
"""

#This is the normal command table, accessible by default
from src.cmdtable import GLOBAL_CMD_TABLE

#The statetable contains sets of cmdtables that is made available
#only when we are in a particular state, overriding the GLOBAL_CMD_TABLE
from src.statetable import GLOBAL_STATE_TABLE

#
# Implementing a simple 'menu' state
#

#the name of our state, to make sure it's the same everywhere
STATENAME = 'menu'

#
# 'entry' command
#
def cmd_entermenu(command):
    """
    This is the 'entry' command that takes the player from the normal
    gameplay mode into the 'menu' state. In order to do this, it
    must be added to the GLOBAL_CMD_TABLE like any command.
    """
    #get the player object calling the command
    source_object = command.source_object
    #this is important: we use the set_state() command
    #to shift the player into a state named 'menu'. Other useful
    #access functions on source_object are get_state()
    # and clear_state(), the latter returns the player to
    # the normal mode of gameplay.
    source_object.set_state(STATENAME)
    #show the menu.
    source_object.execute_cmd('menu')

#
# Commands only available while in the 'menu' state. Note that
#  these have auto_help, so the __doc__ strings of the functions
#  can be read as help entries when in the menu.
#
def menu_cmd_option1(command):
    """
    option1
    This command, obviously,  selects the first option.
    """
    source_object = command.source_object
    print_menu(source_object, 1)
def menu_cmd_option2(command):
    """
    option2
    This command selects the second option. Duh.
    """
    source_object = command.source_object
    print_menu(source_object, 2)
def menu_cmd_menu(command):
    """
    menu
    Clears the options and redraws the menu.
    <<TOPIC:autohelp>>
    This is an extra topic to test the auto-help functionality. The state-help
    system supports nested ('related') topics just like the normal help index does.
    """
    source_object = command.source_object
    print_menu(source_object)

#
# helper function
#
def print_menu(source_obj,choice=None):
    """
    Utility function to print the menu. More interesting things
    would happen here in a real menu.
    """
    if choice==1:
        ch = "%s> option1\n  %soption2" % ('%ch%cy','%cn%cy') #ansi colouring; see src.ansi
    elif choice==2:
        ch = "  %soption1\n%s> option2" % ('%cn%cy','%ch%cy')
    else:
        ch = "  %soption1\n  option2" % ('%cn%cy')

    s ="%sMenu: \n%s\n  %shelp" % ('%ch%cr',ch,'%cn%cy')
    source_obj.emit_to(s)

#Add the 'entry' command to the normal command table
GLOBAL_CMD_TABLE.add_command("entermenu", cmd_entermenu)

#create the state.
GLOBAL_STATE_TABLE.add_state(STATENAME)

#Add the menu commands to the state table by tying them to the 'menu' state. It is
#important that the name of the state matches what we set the player-object to in
#the 'entry' command. Since auto_help is on, we will have help entries for all commands
#while in the menu.
GLOBAL_STATE_TABLE.add_command(STATENAME, "option1", menu_cmd_option1,auto_help=True)
GLOBAL_STATE_TABLE.add_command(STATENAME, "option2", menu_cmd_option2,auto_help=True)
GLOBAL_STATE_TABLE.add_command(STATENAME, "menu", menu_cmd_menu,auto_help=True)

#-----------------------testing the depth of the state system
# This is a test suite that shows off all the features of the state system.
# It sets up a test command @test_state that takes an argument 1-6 for moving into states
# with different characteristics. Note that the only difference as to how the
# various states are created lies in the options given to the add_state() function.
# Use @exit to leave any state.
#
#  All states includes a small test function named "test".
#  1: A very limited state; only contains the "test" command.
#  2: All global commands are included (so this should be the same as normal operation,
#     except you cannot traverse exits and use object-based cmds)
#  3: Only the global commands "get" and "inventory" are included into the state.
#  4: All global commands /except/ "get" and "inventory" are available
#  5: All global commands availabe + ability to traverse exits (not use object-based cmds).
#  6: Only the "test" command, but ability to both traverse exits and use object-based cmds.
#
# Examples of in-game use:
# 1: was used for the menu system above.
# 2: could be used in order to stop someone from moving despite exits being open (tied up?)
# 3: someone incapacitated or blinded might get only limited commands available
# 4: in e.g. a combat state, things like crafting should not be possible
# 5: Pretty much default operation, maybe limiting the use of magical weapons in a room etc?
# 6: A state of panic? You can move, but not take in your surroundings?
#   ... the possibilities are endless.

#defining the test-state names so they are the same everywhere
TSTATE1 = 'no_globals'
TSTATE2 = 'all_globals'
TSTATE3 = 'include_some_globals'
TSTATE4 = 'exclude_some_globals'
TSTATE5 = 'global_allow_exits'
TSTATE6 = 'noglobal_allow_exits_obj_cmds'

#the test command @test_state
def cmd_test_state(command):
    "testing the new state system"
    source_object = command.source_object
    args = command.command_argument
    if not args:
        source_object.emit_to("Usage: @test_state [1..6]")
        return
    arg = args.strip()
    if arg=='1':
        state = TSTATE1
    elif arg=='2':
        state = TSTATE2
    elif arg=='3':
        state = TSTATE3
    elif arg=='4':
        state = TSTATE4
    elif arg=='5':
        state = TSTATE5
    elif arg=='6':
        state = TSTATE6
    else:
        source_object.emit_to("Usage: @test_state [1..6]")
        return
    #set the state
    source_object.set_state(state)
    source_object.emit_to("Now in state '%s' ..." % state)

#a simple command to include in all states.
def cmd_instate_cmd(command):
    "test command in state"
    command.source_object.emit_to("This command works!")

#define some global commands to filter for
cmdfilter = ['get','inventory']

#1: A simple, basic state
GLOBAL_STATE_TABLE.add_state(TSTATE1,exit_command=True)
#2: Include all normal commands in the state
GLOBAL_STATE_TABLE.add_state(TSTATE2,exit_command=True,global_cmds='all')
#3: Include only the two global commands in cmdfilter
GLOBAL_STATE_TABLE.add_state(TSTATE3,exit_command=True,
                             global_cmds='include',global_filter=cmdfilter)
#4: Include all global commands except the ones in cmdfilter
GLOBAL_STATE_TABLE.add_state(TSTATE4,exit_command=True,
                             global_cmds='exclude',global_filter=cmdfilter)
#5: Include all global commands + ability to traverse exits
GLOBAL_STATE_TABLE.add_state(TSTATE5,exit_command=True,
                             global_cmds='all',
                             allow_exits=True)
#6: No global commands, allow exits and commands defined on objects.
GLOBAL_STATE_TABLE.add_state(TSTATE6,exit_command=True,
                             allow_exits=True,allow_obj_cmds=True)

#append the "test" function to all states
GLOBAL_STATE_TABLE.add_command(TSTATE1,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE2,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE3,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE4,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE5,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE6,'test',cmd_instate_cmd)

#create the entry function for testing all states
GLOBAL_CMD_TABLE.add_command('@test_state',cmd_test_state)

<commit_msg>Made the StateSystem example commands in gamesrc/commands/examples/state_example more elaborate and informative. These are the commands:  entermenu (shows a simple menu system)  enterstate (enters one of six different states with descriptions and hints of usage) Leave all states with @exit. /Griatch<commit_after>"""
Example of using the state system. The Event system allows a player object to be
'trapped' in a special environment where different commands are available than normal.
This is very useful in order to implement anything from menus to npc-conversational
choices and inline text-editors.

This example uses the State system to create a simple menu.

To test out this example, add this module to the CUSTOM_COMMAND_MODULES tuple in
your game/settings.py as 'game.gamesrc.commands.examples.state_example' (see ./example.py
for another example). You need to restart the Evennia server before new files are
recognized.

Next enter the mud and give the command
> entermenu

Note that the help entries added to the state system with the auto_help flag are NOT
part of the normal help database, they are stored with the state and only accessible
from inside it (unless you also set the 'global_help' flag in the add_command(), in
which case it is also added to the global help system). If you want to describe the
state itself in more detail you should add that to the main help index manually.

To further test the state system, try the command
> enterstate
"""

#This is the normal command table, accessible by default
from src.cmdtable import GLOBAL_CMD_TABLE

#The statetable contains sets of cmdtables that is made available
#only when we are in a particular state, overriding the GLOBAL_CMD_TABLE
from src.statetable import GLOBAL_STATE_TABLE

#
# Implementing a simple 'menu' state
#

#the name of our state, to make sure it's the same everywhere
STATENAME = 'menu'

#
# 'entry' command
#
def cmd_entermenu(command):
    """
    This is the 'entry' command that takes the player from the normal
    gameplay mode into the 'menu' state. In order to do this, it
    must be added to the GLOBAL_CMD_TABLE like any command.
    """
    #get the player object calling the command
    source_object = command.source_object
    #this is important: we use the set_state() command
    #to shift the player into a state named 'menu'. Other useful
    #access functions on source_object are get_state()
    # and clear_state(), the latter returns the player to
    # the normal mode of gameplay.
    source_object.set_state(STATENAME)
    #show the menu.
    s = """
     Welcome to the Demo menu!
     In this demo all you can do is select one of the two options so it changes colour.
     This is just intended to show off the possibility of the state system. More
     interesting things should of course happen in a real menu. Use @exit to
     leave the menu."""
    source_object.emit_to(s)
    source_object.execute_cmd('menu')

#
# Commands only available while in the 'menu' state. Note that
#  these have auto_help, so the __doc__ strings of the functions
#  can be read as help entries when in the menu.
#
def menu_cmd_option1(command):
    """
    option1
    This command, obviously,  selects the first option.
    """
    source_object = command.source_object
    print_menu(source_object, 1)
def menu_cmd_option2(command):
    """
    option2
    This command selects the second option. Duh.
    """
    source_object = command.source_object
    print_menu(source_object, 2)
def menu_cmd_menu(command):
    """
    menu
    Clears the options and redraws the menu.
    <<TOPIC:autohelp>>
    This is an extra topic to test the auto-help functionality. The state-help
    system supports nested ('related') topics just like the normal help index does.
    """
    source_object = command.source_object
    print_menu(source_object)

#
# helper function
#
def print_menu(source_obj,choice=None):
    """
    Utility function to print the menu. More interesting things
    would happen here in a real menu.
    """

    if choice==1:
        ch = "%s> option1\n  %soption2" % ('%ch%cy','%cn%cy') #ansi colouring; see src.ansi
    elif choice==2:
        ch = "  %soption1\n%s> option2" % ('%cn%cy','%ch%cy')
    else:
        ch = "  %soption1\n  option2" % ('%cn%cy')

    s ="\n%sMenu: \n%s\n  %shelp" % ('%ch%cr',ch,'%cn%cy')
    source_obj.emit_to(s)

#Add the 'entry' command to the normal command table
GLOBAL_CMD_TABLE.add_command("entermenu", cmd_entermenu)

#create the state.
GLOBAL_STATE_TABLE.add_state(STATENAME,exit_command=True)

#Add the menu commands to the state table by tying them to the 'menu' state. It is
#important that the name of the state matches what we set the player-object to in
#the 'entry' command. Since auto_help is on, we will have help entries for all commands
#while in the menu.
GLOBAL_STATE_TABLE.add_command(STATENAME, "option1", menu_cmd_option1,auto_help=True)
GLOBAL_STATE_TABLE.add_command(STATENAME, "option2", menu_cmd_option2,auto_help=True)
GLOBAL_STATE_TABLE.add_command(STATENAME, "menu", menu_cmd_menu,auto_help=True)

#-----------------------testing the depth of the state system
# This is a test suite that shows off all the features of the state system.
# It sets up a test command @test_state that takes an argument 1-6 for moving into states
# with different characteristics. Note that the only difference as to how the
# various states are created lies in the options given to the add_state() function.
# Use @exit to leave any state.
#
#  All states includes a small test function named "test".
#  1: A very limited state; only contains the "test" command.
#  2: All global commands are included (so this should be the same as normal operation,
#     except you cannot traverse exits and use object-based cmds)
#  3: Only the global commands "get" and "inventory" are included into the state.
#  4: All global commands /except/ "get" and "inventory" are available
#  5: All global commands availabe + ability to traverse exits (not use object-based cmds).
#  6: Only the "test" command, but ability to both traverse exits and use object-based cmds.
#
# Examples of in-game use:
# 1: was used for the menu system above.
# 2: could be used in order to stop someone from moving despite exits being open (tied up?)
# 3: someone incapacitated or blinded might get only limited commands available
# 4: in e.g. a combat state, things like crafting should not be possible
# 5: Pretty much default operation, maybe limiting the use of magical weapons in a room etc?
# 6: A state of panic? You can move, but not take in your surroundings?
#   ... the possibilities are endless.

#defining the test-state names so they are the same everywhere
TSTATE1 = 'no_globals'
TSTATE2 = 'all_globals'
TSTATE3 = 'include_some_globals'
TSTATE4 = 'exclude_some_globals'
TSTATE5 = 'global_allow_exits'
TSTATE6 = 'noglobal_allow_exits_obj_cmds'

#
#the test command @test_state
#
def cmd_test_state(command):
    "testing the new state system"
    source_object = command.source_object
    args = command.command_argument
    if not args:
        source_object.emit_to("Usage: enterstate 1 - 6")
        return
    s = "\n Entering state ... \nThis state includes the commands 'test', 'help', '@exit' and "
    arg = args.strip()
    if arg=='1':
        s += "no global commands at all. With some more state commands, this state would work well for e.g. a combat state or a menu where the player don't need access to the normal command definitions. Take a special look at the help command, which is in fact a state-only version of the normal help."
        state = TSTATE1
    elif arg=='2':
        s += "all global commands. You should be able to do everything as normal, but not move around."
        state = TSTATE2
    elif arg=='3':
        s += "the global commands 'inv' and 'get' only."
        state = TSTATE3
    elif arg=='4':
        s += "all global commands *except* 'inv' and 'get' (try using them). This allows you to disable commands that should not be possible at a certain time (like starting to craft while in a fight or something)."
        state = TSTATE4
    elif arg=='5':
        s += "all global commands as well as the ability to traverse exits. You do not have the ability to use commands defined on objects though."
        state = TSTATE5
    elif arg=='6':
        s += "no globals at all, but you have the ability to both use exits and commands on items. This would maybe be interesting for a 'total darkness' state or maybe a 'panic' state where you can move around but cannot actually take in your surroundings."
        state = TSTATE6
    else:
        source_object.emit_to("Usage: enterstate 1 - 6")
        return
    #set the state
    source_object.set_state(state)
    source_object.emit_to("%s\n (Now in state %s: '%s' ... use @exit to leave the state.)" % (s,arg,state))

#a simple command to include in all states.
def cmd_instate_cmd(command):
    """
    This is the help text for the test command (created with the auto_help sytem).
    This is a state-only command that does not exist outside this state. Since this state
    is completely isolated from the normal gameplay, commands can also
    harmlessly redefine any normal command - so if there was a normal command named
    'test', it would remain unchanged when we leave the state.
    """
    command.source_object.emit_to("This state command works!")

#define some global commands to filter for
cmdfilter = ['get','inventory']

#1: A simple, basic state
GLOBAL_STATE_TABLE.add_state(TSTATE1,exit_command=True)
#2: Include all normal commands in the state
GLOBAL_STATE_TABLE.add_state(TSTATE2,exit_command=True,global_cmds='all')
#3: Include only the two global commands in cmdfilter
GLOBAL_STATE_TABLE.add_state(TSTATE3,exit_command=True,
                             global_cmds='include',global_filter=cmdfilter)
#4: Include all global commands except the ones in cmdfilter
GLOBAL_STATE_TABLE.add_state(TSTATE4,exit_command=True,
                             global_cmds='exclude',global_filter=cmdfilter)
#5: Include all global commands + ability to traverse exits
GLOBAL_STATE_TABLE.add_state(TSTATE5,exit_command=True,
                             global_cmds='all',
                             allow_exits=True)
#6: No global commands, allow exits and commands defined on objects.
GLOBAL_STATE_TABLE.add_state(TSTATE6,exit_command=True,
                             allow_exits=True,allow_obj_cmds=True)

#append the "test" function to all states
GLOBAL_STATE_TABLE.add_command(TSTATE1,'test',cmd_instate_cmd,auto_help=True)
GLOBAL_STATE_TABLE.add_command(TSTATE2,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE3,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE4,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE5,'test',cmd_instate_cmd)
GLOBAL_STATE_TABLE.add_command(TSTATE6,'test',cmd_instate_cmd)

#create the entry function for testing all states
GLOBAL_CMD_TABLE.add_command('enterstate',cmd_test_state)

<|endoftext|>


<commit_before>import os
from nose.tools import assert_equals
from corpkit import *

unparsed_path = 'data/test'
parsed_path = 'data/test-stripped-parsed'

def test_import():
    import corpkit
    from dictionaries.wordlists import wordlists
    from corpkit import Corpus

def test_corpus_class():
    unparsed = Corpus(unparsed_path)
    assert_equals(os.path.basename(unparsed_path), unparsed.name)

def test_parse():
    print 'Testing parser'
    unparsed = Corpus(unparsed_path)
    parsed = unparsed.parse()
    assert_equals(list([i.name for i in parsed.files]), ['intro.txt.xml', 'body.txt.xml'])

test_parse.slow = 1

def test_parse_speakseg(skipassert = False):
    print 'Testing parser with speaker segmentation'
    unparsed = Corpus(unparsed_path)
    parsed = unparsed.parse(speaker_segmentation = True)
    if skipassert:
        assert_equals(list([i.name for i in parsed.files]), ['intro.txt.xml', 'body.txt.xml'])

test_parse_speakseg.slow = 1

if os.path.isdir(parsed_path):
    corp = Corpus('data/test-stripped-parsed')
else:
    test_parse_speakseg(skipassert = True)

def test_interro1():
    print 'Testing interrogation 1'

    data = corp.interrogate('t', r'__ < /JJ.?/')
    assert_equals(data.results.shape, (2, 5))

def test_interro2():
    print 'Testing interrogation 2'
    corp = Corpus('data/test-parsed')
    data = corp.interrogate({'t': r'__ < /JJ.?/'})
    assert_equals(data.results.shape, (2, 5))

#def test_interro3():
#    print 'Testing interrogation 3'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro4():
#    print 'Testing interrogation 4'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro5():
#    print 'Testing interrogation 5'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro6():
#    print 'Testing interrogation 6'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro7():
#    print 'Testing interrogation 7'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro8():
#    print 'Testing interrogation 8'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_interro9():
#    print 'Testing interrogation 9'
#    corp = Corpus('data/test-parsed')
#    tot = False
#    assert_equals(tot, tot)
#
#def test_edit1():
#    print 'Testing edit'
#    tot = False
#    assert_equals(tot, tot)

def test_interro_multiindex_tregex_justspeakers():
    import pandas as pd
    data = corp.interrogate('t', r'__ < /JJ.?/', just_speakers = ['each'])
    assert_equals(all(data.multiindex().index), all(pd.MultiIndex(levels=[[u'ANONYMOUS', u'TESTER', u'Total'],
           [u'first', u'second', u'Total']],
           labels=[[0, 0, 1, 1], [0, 1, 0, 1]],
           names=[u'corpus', u'subcorpus'])))
<commit_msg>useful set of tests<commit_after># full suite: must delete some corpora
# nosetests corpkit/nosetests.py

# skip parsing
# nosetests corpkit/nosetests.py -a '!slow'

import os
from nose.tools import assert_equals
from corpkit import *

unparsed_path = 'data/test'
parsed_path = 'data/test-stripped-parsed'

def test_import():
    import corpkit
    from dictionaries.wordlists import wordlists
    from corpkit import Corpus
    assert_equals(wordlists.articles, [u'a', u'an', u'the', u'teh'])

def test_corpus_class():
    unparsed = Corpus(unparsed_path)
    assert_equals(os.path.basename(unparsed_path), unparsed.name)

def test_parse():
    print 'Testing parser'
    unparsed = Corpus(unparsed_path)
    parsed = unparsed.parse()
    assert_equals(list([i.name for i in parsed.files]), ['intro.txt.xml', 'body.txt.xml'])

test_parse.slow = 1

def test_parse_speakseg(skipassert = False):
    print 'Testing parser with speaker segmentation'
    unparsed = Corpus(unparsed_path)
    parsed = unparsed.parse(speaker_segmentation = True)
    if skipassert:
        assert_equals(list([i.name for i in parsed.files]), ['intro.txt.xml', 'body.txt.xml'])

test_parse_speakseg.slow = 1

if os.path.isdir(parsed_path):
    corp = Corpus('data/test-stripped-parsed')
else:
    test_parse_speakseg(skipassert = True)

def test_interro1():
    print 'Testing interrogation 1'
    data = corp.interrogate('t', r'__ < /JJ.?/')
    assert_equals(data.results.shape, (2, 5))

def test_interro2():
    print 'Testing interrogation 2'
    data = corp.interrogate({'t': r'__ < /JJ.?/'})
    assert_equals(data.results.shape, (2, 5))

def test_interro3():
    print 'Testing interrogation 3'
    data = corp.interrogate({'w': r'^c'}, exclude = {'l': r'check'}, show = ['l', 'f'])
    lst = ['corpus/nsubjpass', 'corpkit/nmod:poss', 'continuum/nmod:on', 'conduit/nmod:as',
           'concordancing/nmod:like', 'computational/amod', 'case/nmod:in', 'can/aux']
    assert_equals(list(data.results.columns), lst)

def test_interro4():
    print 'Testing interrogation 4'
    corp = Corpus('data/test-stripped-tokenised')
    data = corp.interrogate({'n': 'any'})
    d = {u'and interrogating': {'first': 0, 'second': 2},
         u'concordancing and': {'first': 0, 'second': 2}}
    assert_equals(data.results.to_dict(), d)

def test_interro5():
    print 'Testing interrogation 5'
    corp = Corpus('data/test-stripped')
    data = corp.interrogate({'w': r'\bl[a-z]+?\s'})
    assert_equals(data.results.sum().sum(), 4)

def test_interro_multiindex_tregex_justspeakers():
    print 'Testing multiindex'
    import pandas as pd
    data = corp.interrogate('t', r'__ < /JJ.?/', just_speakers = ['each'])
    assert_equals(all(data.multiindex().index), all(pd.MultiIndex(levels=[[u'ANONYMOUS',
           u'NEWCOMER', u'TESTER', u'UNIDENTIFIED', u'Total'], [u'first', u'second', u'Total']],
           labels=[[3, 3, 1, 1, 0, 0, 2, 2], [0, 1, 0, 1, 0, 1, 0, 1]],
           names=[u'corpus', u'subcorpus'])))

def test_conc():
    print 'Testing concordancer'
    data = corp.concordance({'f': 'amod'})
    assert_equals('corpus linguistics and', data.ix[0][3])

def test_edit():
    print 'Testing simple edit'
    data = corp.interrogate({'t': r'__ !< __'})
    data = data.edit('%', 'self')
    assert_equals(data.results.iloc[0,0], 8.1081081081081088)

<|endoftext|>


<commit_before>from unidecode import unidecode
from celery.decorators import task
from couchexport.shortcuts import get_export_files
from django.core.cache import cache
import uuid
from soil import CachedDownload, FileDownload
from couchexport.models import Format
import tempfile
import os
import stat

@task
def export_async(download_id, export_tag, format=None, filename=None,
                 previous_export_id=None, filter=None,
                 expiry=10*60*60):

    (tmp, checkpoint) = get_export_files(export_tag, format, previous_export_id, filter)
    if checkpoint:
        temp_id = uuid.uuid4().hex
        fd, path = tempfile.mkstemp()
        with os.fdopen(fd, 'wb') as file:
            file.write(tmp.getvalue())
        # make file globally read/writeable in case celery runs as root
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | \
                 stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
        format = Format.from_format(format)
        cache.set(download_id, FileDownload(path, mimetype=format.mimetype,
                                            content_disposition='attachment; filename=%s.%s' % \
                                            (unidecode(filename), format.extension),
                                            extras={'X-CommCareHQ-Export-Token': checkpoint.get_id}))
    else:
        temp_id = uuid.uuid4().hex
        cache.set(temp_id, "Sorry, there wasn't any data.", expiry)
        cache.set(download_id, CachedDownload(temp_id,content_disposition="",
                                              mimetype="text/html"), expiry)

<commit_msg>properly set expiry on download<commit_after>from unidecode import unidecode
from celery.decorators import task
from couchexport.shortcuts import get_export_files
from django.core.cache import cache
import uuid
from soil import CachedDownload, FileDownload
from couchexport.models import Format
import tempfile
import os
import stat

@task
def export_async(download_id, export_tag, format=None, filename=None,
                 previous_export_id=None, filter=None,
                 expiry=10*60*60):

    (tmp, checkpoint) = get_export_files(export_tag, format, previous_export_id, filter)
    if checkpoint:
        temp_id = uuid.uuid4().hex
        fd, path = tempfile.mkstemp()
        with os.fdopen(fd, 'wb') as file:
            file.write(tmp.getvalue())
        # make file globally read/writeable in case celery runs as root
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | \
                 stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
        format = Format.from_format(format)
        cache.set(download_id, FileDownload(path, mimetype=format.mimetype,
                                            content_disposition='attachment; filename=%s.%s' % \
                                            (unidecode(filename), format.extension),
                                            extras={'X-CommCareHQ-Export-Token': checkpoint.get_id}),
                                            expiry)
    else:
        temp_id = uuid.uuid4().hex
        cache.set(temp_id, "Sorry, there wasn't any data.", expiry)
        cache.set(download_id, CachedDownload(temp_id,content_disposition="",
                                              mimetype="text/html"), expiry)

<|endoftext|>