<commit_before>"""
The magics offered by the HoloViews IPython extension are powerful and
support rich, compositional specifications. To avoid the the brittle,
convoluted code that results from trying to support the syntax in pure
Python, this file defines suitable parsers using pyparsing that are
cleaner and easier to understand.

Pyparsing is required by matplotlib and will therefore be available if
HoloViews is being used in conjunction with matplotlib.
"""
import param
from itertools import groupby
import pyparsing as pp

from holoviews.core.options import Options
from ..operation import Compositor

ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&\()*+,-./:;<=>?@\\^_`{|}~'

# To generate warning in the standard param style
# Parameterize Parser and use warning method once param supports
# logging at the class level.
class ParserWarning(param.Parameterized):pass
parsewarning = ParserWarning(name='Warning')

class Parser(object):
    """
    Base class for magic line parsers, designed for forgiving parsing
    of keyword lists.
    """

    # Static namespace set in __init__.py of the extension
    namespace = {}
    # If True, raise SyntaxError on eval error otherwise warn
    abort_on_eval_failure = False

    @classmethod
    def _strip_commas(cls, kw):
        "Strip out any leading/training commas from the token"
        kw = kw[:-1] if kw[-1]==',' else kw
        return kw[1:] if kw[0]==',' else kw

    @classmethod
    def collect_tokens(cls, parseresult, mode):
        """
        Collect the tokens from a (potentially) nested parse result.
        """
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                tokens[-1] = tokens[-1] + (inner % ''.join(token))
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens

    @classmethod
    def todict(cls, parseresult, mode='parens', ns={}):
        """
        Helper function to return dictionary given the parse results
        from a pyparsing.nestedExpr object (containing keywords).

        The ns is a dynamic namespace (typically the IPython Notebook
        namespace) used to update the class-level namespace.
        """
        grouped, kwargs = [], {}
        tokens = cls.collect_tokens(parseresult, mode)
        # Group tokens without '=' and append to last token containing '='
        for group in groupby(tokens, lambda el: '=' in el):
            (val, items) = group
            if val is True:
                grouped += list(items)
            if val is False:
                elements =list(items)
                # Assume anything before ) or } can be joined with commas
                # (e.g tuples with spaces in them)
                joiner=',' if any(((')' in el) or ('}' in el))
                                  for el in elements) else ''
                grouped[-1] += joiner + joiner.join(elements)

        for keyword in grouped:
            # Tuple ('a', 3) becomes (,'a',3) and '(,' is never valid
            # Same for some of the other joining errors corrected here
            for (fst,snd) in [('(,', '('), ('{,', '{'), ('=,','='),
                              (',:',':'), (':,', ':'), (',,', ',')]:
                keyword = keyword.replace(fst, snd)
            try:
                kwargs.update(eval('dict(%s)' % keyword,
                                   dict(cls.namespace, **ns)))
            except:
                if cls.abort_on_eval_failure:
                    raise SyntaxError("Could not evaluate keyword: %r"
                                      % keyword)
                msg = "Ignoring keyword pair that fails to evaluate: '%s'"
                parsewarning.warning(msg % keyword)

        return kwargs

class OptsSpec(Parser):
    """
    An OptsSpec is a string specification that describes an
    OptionTree. It is a list of tree path specifications (using dotted
    syntax) separated by keyword lists for any of the style, plotting
    or normalization options. These keyword lists are denoted
    'plot(..)', 'style(...)' and 'norm(...)'  respectively.  These
    three groups may be specified even more concisely using keyword
    lists delimited by square brackets, parentheses and braces
    respectively.  All these sets are optional and may be supplied in
    any order.

    For instance, the following string:

    Image (interpolation=None) plot(show_title=False) Curve style(color='r')

    Would specify an OptionTree where Image has "interpolation=None"
    for style and 'show_title=False' for plot options. The Curve has a
    style set such that color='r'.

    The parser is fairly forgiving; commas between keywords are
    optional and additional spaces are often allowed. The only
    restriction is that keywords *must* be immediately followed by the
    '=' sign (no space).
    """

    plot_options_short = pp.nestedExpr('[',
                                       ']',
                                       content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                                   ).setResultsName('plot_options')

    plot_options_long = pp.nestedExpr(opener='plot[',
                                      closer=']',
                                      content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                                  ).setResultsName('plot_options')

    plot_options = (plot_options_short | plot_options_long)

    style_options_short = pp.nestedExpr(opener='(',
                                        closer=')',
                                        ignoreExpr=None
                                    ).setResultsName("style_options")

    style_options_long = pp.nestedExpr(opener='style(',
                                       closer=')',
                                       ignoreExpr=None
                                   ).setResultsName("style_options")

    style_options = (style_options_short | style_options_long)

    norm_options_short = pp.nestedExpr(opener='{',
                                       closer='}',
                                       ignoreExpr=None
                                   ).setResultsName("norm_options")

    norm_options_long = pp.nestedExpr(opener='norm{',
                                      closer='}',
                                      ignoreExpr=None
                                  ).setResultsName("norm_options")

    norm_options = (norm_options_short | norm_options_long)

    compositor_ops = pp.MatchFirst(
        [pp.Literal(el.group) for el in Compositor.definitions])

    dotted_path = pp.Combine( pp.Word(ascii_uppercase, exact=1)
                              + pp.Word(pp.alphanums+'._'))

    pathspec = (dotted_path | compositor_ops).setResultsName("pathspec")

    spec_group = pp.Group(pathspec +
                          (pp.Optional(norm_options)
                           & pp.Optional(plot_options)
                           & pp.Optional(style_options)))

    opts_spec = pp.OneOrMore(spec_group)

    # Aliases that map to the current option name for backward compatibility
    aliases = {'horizontal_spacing':'hspace',
               'vertical_spacing':  'vspace',
               'figure_alpha':'    fig_alpha',
               'figure_bounds':   'fig_bounds',
               'figure_inches':   'fig_inches',
               'figure_latex':    'fig_latex',
               'figure_rcparams': 'fig_rcparams',
               'figure_size':     'fig_size',
               'show_xaxis':      'xaxis',
               'show_yaxis':      'yaxis'}

    @classmethod
    def process_normalization(cls, parse_group):
        """
        Given a normalization parse group (i.e. the contents of the
        braces), validate the option list and compute the appropriate
        integer value for the normalization plotting option.
        """
        if ('norm_options' not in parse_group): return None
        opts = parse_group['norm_options'][0].asList()
        if opts == []: return None

        options = ['+framewise', '-framewise', '+axiswise', '-axiswise']

        for normopt in options:
            if opts.count(normopt) > 1:
                raise SyntaxError("Normalization specification must not"
                                  " contain repeated %r" % normopt)

        if not all(opt in options for opt in opts):
            raise SyntaxError("Normalization option not one of %s"
                              % ", ".join(options))
        excluded = [('+framewise', '-framewise'), ('+axiswise', '-axiswise')]
        for pair in excluded:
            if all(exclude in opts for exclude in pair):
                raise SyntaxError("Normalization specification cannot"
                                  " contain both %s and %s" % (pair[0], pair[1]))

        # If unspecified, default is -axiswise and -framewise
        if len(opts) == 1 and opts[0].endswith('framewise'):
            axiswise = False
            framewise = True if '+framewise' in opts else False
        elif len(opts) == 1 and opts[0].endswith('axiswise'):
            framewise = False
            axiswise = True if '+axiswise' in opts else False
        else:
            axiswise = True if '+axiswise' in opts else False
            framewise = True if '+framewise' in opts else False

        return dict(axiswise=axiswise,
                    framewise=framewise)

    @classmethod
    def parse(cls, line, ns={}):
        """
        Parse an options specification, returning a dictionary with
        path keys and {'plot':<options>, 'style':<options>} values.
        """
        parses  = [p for p in cls.opts_spec.scanString(line)]
        if len(parses) != 1:
            raise SyntaxError("Invalid specification syntax.")
        else:
            e = parses[0][2]
            processed = line[:e]
            if (processed.strip() != line.strip()):
                raise SyntaxError("Failed to parse remainder of string: %r" % line[e:])

        parse = {}
        for group in cls.opts_spec.parseString(line):
            options = {}

            normalization = cls.process_normalization(group)
            if normalization is not None:
                options['norm'] = Options(**normalization)

            if 'plot_options' in group:
                plotopts =  group['plot_options'][0]
                opts = cls.todict(plotopts, 'brackets', ns=ns)
                options['plot'] = Options(**{cls.aliases.get(k,k):v for k,v in opts.items()})

            if 'style_options' in group:
                styleopts = group['style_options'][0]
                opts = cls.todict(styleopts, 'parens', ns=ns)
                options['style'] = Options(**{cls.aliases.get(k,k):v for k,v in opts.items()})

            if group['pathspec'] in parse:
                # Update in case same pathspec accidentally repeated by the user.
                parse[group['pathspec']].update(options)
            else:
                parse[group['pathspec']] = options
        return parse

class CompositorSpec(Parser):
    """
    The syntax for defining a set of compositor is as follows:

    [ mode op(spec) [settings] value ]+

    The components are:

    mode      : Operation mode, either 'data' or 'display'.
    group     : Value identifier with capitalized initial letter.
    op        : The name of the operation to apply.
    spec      : Overlay specification of form (A * B) where A and B are
                 dotted path specifications.
    settings  : Optional list of keyword arguments to be used as
                parameters to the operation (in square brackets).
    """

    mode = pp.Word(pp.alphas+pp.nums+'_').setResultsName("mode")

    op = pp.Word(pp.alphas+pp.nums+'_').setResultsName("op")

    overlay_spec = pp.nestedExpr(opener='(',
                                 closer=')',
                                 ignoreExpr=None
                             ).setResultsName("spec")

    value = pp.Word(pp.alphas+pp.nums+'_').setResultsName("value")

    op_settings = pp.nestedExpr(opener='[',
                                closer=']',
                                ignoreExpr=None
                            ).setResultsName("op_settings")

    compositor_spec = pp.OneOrMore(pp.Group(mode + op + overlay_spec + value
                                            + pp.Optional(op_settings)))

    @classmethod
    def parse(cls, line, ns={}):
        """
        Parse compositor specifications, returning a list Compositors
        """
        definitions = []
        parses  = [p for p in cls.compositor_spec.scanString(line)]
        if len(parses) != 1:
            raise SyntaxError("Invalid specification syntax.")
        else:
            e = parses[0][2]
            processed = line[:e]
            if (processed.strip() != line.strip()):
                raise SyntaxError("Failed to parse remainder of string: %r" % line[e:])

        opmap = {op.__name__:op for op in Compositor.operations}
        for group in cls.compositor_spec.parseString(line):

            if ('mode' not in group) or group['mode'] not in ['data', 'display']:
                raise SyntaxError("Either data or display mode must be specified.")
            mode = group['mode']

            kwargs = {}
            operation = opmap[group['op']]
            spec = ' '.join(group['spec'].asList()[0])

            if  group['op'] not in opmap:
                raise SyntaxError("Operation %s not available for use with compositors."
                                  % group['op'])
            if  'op_settings' in group:
                kwargs = cls.todict(group['op_settings'][0], 'brackets', ns=ns)

            definition = Compositor(str(spec), operation, str(group['value']), mode, **kwargs)
            definitions.append(definition)
        return definitions
<commit_msg>Fix for parsing of nested tokens<commit_after>"""
The magics offered by the HoloViews IPython extension are powerful and
support rich, compositional specifications. To avoid the the brittle,
convoluted code that results from trying to support the syntax in pure
Python, this file defines suitable parsers using pyparsing that are
cleaner and easier to understand.

Pyparsing is required by matplotlib and will therefore be available if
HoloViews is being used in conjunction with matplotlib.
"""
import param
from itertools import groupby
import pyparsing as pp

from holoviews.core.options import Options
from ..operation import Compositor

ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!#$%&\()*+,-./:;<=>?@\\^_`{|}~'

# To generate warning in the standard param style
# Parameterize Parser and use warning method once param supports
# logging at the class level.
class ParserWarning(param.Parameterized):pass
parsewarning = ParserWarning(name='Warning')

class Parser(object):
    """
    Base class for magic line parsers, designed for forgiving parsing
    of keyword lists.
    """

    # Static namespace set in __init__.py of the extension
    namespace = {}
    # If True, raise SyntaxError on eval error otherwise warn
    abort_on_eval_failure = False

    @classmethod
    def _strip_commas(cls, kw):
        "Strip out any leading/training commas from the token"
        kw = kw[:-1] if kw[-1]==',' else kw
        return kw[1:] if kw[0]==',' else kw

    @classmethod
    def recurse_token(cls, token, inner):
        recursed = []
        for tok in token:
            if isinstance(tok, list):
                new_tok = [recurse_token(t, inner) if isinstance(t, list) else t
                           for t in tok]
                recursed.append((inner % ''.join(new_tok)))
            else:
                recursed.append(tok)
        return recursed

    @classmethod
    def collect_tokens(cls, parseresult, mode):
        """
        Collect the tokens from a (potentially) nested parse result.
        """
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + (inner % ''.join(token))
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens

    @classmethod
    def todict(cls, parseresult, mode='parens', ns={}):
        """
        Helper function to return dictionary given the parse results
        from a pyparsing.nestedExpr object (containing keywords).

        The ns is a dynamic namespace (typically the IPython Notebook
        namespace) used to update the class-level namespace.
        """
        grouped, kwargs = [], {}
        tokens = cls.collect_tokens(parseresult, mode)
        # Group tokens without '=' and append to last token containing '='
        for group in groupby(tokens, lambda el: '=' in el):
            (val, items) = group
            if val is True:
                grouped += list(items)
            if val is False:
                elements =list(items)
                # Assume anything before ) or } can be joined with commas
                # (e.g tuples with spaces in them)
                joiner=',' if any(((')' in el) or ('}' in el))
                                  for el in elements) else ''
                grouped[-1] += joiner + joiner.join(elements)

        for keyword in grouped:
            # Tuple ('a', 3) becomes (,'a',3) and '(,' is never valid
            # Same for some of the other joining errors corrected here
            for (fst,snd) in [('(,', '('), ('{,', '{'), ('=,','='),
                              (',:',':'), (':,', ':'), (',,', ',')]:
                keyword = keyword.replace(fst, snd)
            try:
                kwargs.update(eval('dict(%s)' % keyword,
                                   dict(cls.namespace, **ns)))
            except:
                if cls.abort_on_eval_failure:
                    raise SyntaxError("Could not evaluate keyword: %r"
                                      % keyword)
                msg = "Ignoring keyword pair that fails to evaluate: '%s'"
                parsewarning.warning(msg % keyword)

        return kwargs

class OptsSpec(Parser):
    """
    An OptsSpec is a string specification that describes an
    OptionTree. It is a list of tree path specifications (using dotted
    syntax) separated by keyword lists for any of the style, plotting
    or normalization options. These keyword lists are denoted
    'plot(..)', 'style(...)' and 'norm(...)'  respectively.  These
    three groups may be specified even more concisely using keyword
    lists delimited by square brackets, parentheses and braces
    respectively.  All these sets are optional and may be supplied in
    any order.

    For instance, the following string:

    Image (interpolation=None) plot(show_title=False) Curve style(color='r')

    Would specify an OptionTree where Image has "interpolation=None"
    for style and 'show_title=False' for plot options. The Curve has a
    style set such that color='r'.

    The parser is fairly forgiving; commas between keywords are
    optional and additional spaces are often allowed. The only
    restriction is that keywords *must* be immediately followed by the
    '=' sign (no space).
    """

    plot_options_short = pp.nestedExpr('[',
                                       ']',
                                       content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                                   ).setResultsName('plot_options')

    plot_options_long = pp.nestedExpr(opener='plot[',
                                      closer=']',
                                      content=pp.OneOrMore(pp.Word(allowed) ^ pp.quotedString)
                                  ).setResultsName('plot_options')

    plot_options = (plot_options_short | plot_options_long)

    style_options_short = pp.nestedExpr(opener='(',
                                        closer=')',
                                        ignoreExpr=None
                                    ).setResultsName("style_options")

    style_options_long = pp.nestedExpr(opener='style(',
                                       closer=')',
                                       ignoreExpr=None
                                   ).setResultsName("style_options")

    style_options = (style_options_short | style_options_long)

    norm_options_short = pp.nestedExpr(opener='{',
                                       closer='}',
                                       ignoreExpr=None
                                   ).setResultsName("norm_options")

    norm_options_long = pp.nestedExpr(opener='norm{',
                                      closer='}',
                                      ignoreExpr=None
                                  ).setResultsName("norm_options")

    norm_options = (norm_options_short | norm_options_long)

    compositor_ops = pp.MatchFirst(
        [pp.Literal(el.group) for el in Compositor.definitions])

    dotted_path = pp.Combine( pp.Word(ascii_uppercase, exact=1)
                              + pp.Word(pp.alphanums+'._'))

    pathspec = (dotted_path | compositor_ops).setResultsName("pathspec")

    spec_group = pp.Group(pathspec +
                          (pp.Optional(norm_options)
                           & pp.Optional(plot_options)
                           & pp.Optional(style_options)))

    opts_spec = pp.OneOrMore(spec_group)

    # Aliases that map to the current option name for backward compatibility
    aliases = {'horizontal_spacing':'hspace',
               'vertical_spacing':  'vspace',
               'figure_alpha':'    fig_alpha',
               'figure_bounds':   'fig_bounds',
               'figure_inches':   'fig_inches',
               'figure_latex':    'fig_latex',
               'figure_rcparams': 'fig_rcparams',
               'figure_size':     'fig_size',
               'show_xaxis':      'xaxis',
               'show_yaxis':      'yaxis'}

    @classmethod
    def process_normalization(cls, parse_group):
        """
        Given a normalization parse group (i.e. the contents of the
        braces), validate the option list and compute the appropriate
        integer value for the normalization plotting option.
        """
        if ('norm_options' not in parse_group): return None
        opts = parse_group['norm_options'][0].asList()
        if opts == []: return None

        options = ['+framewise', '-framewise', '+axiswise', '-axiswise']

        for normopt in options:
            if opts.count(normopt) > 1:
                raise SyntaxError("Normalization specification must not"
                                  " contain repeated %r" % normopt)

        if not all(opt in options for opt in opts):
            raise SyntaxError("Normalization option not one of %s"
                              % ", ".join(options))
        excluded = [('+framewise', '-framewise'), ('+axiswise', '-axiswise')]
        for pair in excluded:
            if all(exclude in opts for exclude in pair):
                raise SyntaxError("Normalization specification cannot"
                                  " contain both %s and %s" % (pair[0], pair[1]))

        # If unspecified, default is -axiswise and -framewise
        if len(opts) == 1 and opts[0].endswith('framewise'):
            axiswise = False
            framewise = True if '+framewise' in opts else False
        elif len(opts) == 1 and opts[0].endswith('axiswise'):
            framewise = False
            axiswise = True if '+axiswise' in opts else False
        else:
            axiswise = True if '+axiswise' in opts else False
            framewise = True if '+framewise' in opts else False

        return dict(axiswise=axiswise,
                    framewise=framewise)

    @classmethod
    def parse(cls, line, ns={}):
        """
        Parse an options specification, returning a dictionary with
        path keys and {'plot':<options>, 'style':<options>} values.
        """
        parses  = [p for p in cls.opts_spec.scanString(line)]
        if len(parses) != 1:
            raise SyntaxError("Invalid specification syntax.")
        else:
            e = parses[0][2]
            processed = line[:e]
            if (processed.strip() != line.strip()):
                raise SyntaxError("Failed to parse remainder of string: %r" % line[e:])

        parse = {}
        for group in cls.opts_spec.parseString(line):
            options = {}

            normalization = cls.process_normalization(group)
            if normalization is not None:
                options['norm'] = Options(**normalization)

            if 'plot_options' in group:
                plotopts =  group['plot_options'][0]
                opts = cls.todict(plotopts, 'brackets', ns=ns)
                options['plot'] = Options(**{cls.aliases.get(k,k):v for k,v in opts.items()})

            if 'style_options' in group:
                styleopts = group['style_options'][0]
                opts = cls.todict(styleopts, 'parens', ns=ns)
                options['style'] = Options(**{cls.aliases.get(k,k):v for k,v in opts.items()})

            if group['pathspec'] in parse:
                # Update in case same pathspec accidentally repeated by the user.
                parse[group['pathspec']].update(options)
            else:
                parse[group['pathspec']] = options
        return parse

class CompositorSpec(Parser):
    """
    The syntax for defining a set of compositor is as follows:

    [ mode op(spec) [settings] value ]+

    The components are:

    mode      : Operation mode, either 'data' or 'display'.
    group     : Value identifier with capitalized initial letter.
    op        : The name of the operation to apply.
    spec      : Overlay specification of form (A * B) where A and B are
                 dotted path specifications.
    settings  : Optional list of keyword arguments to be used as
                parameters to the operation (in square brackets).
    """

    mode = pp.Word(pp.alphas+pp.nums+'_').setResultsName("mode")

    op = pp.Word(pp.alphas+pp.nums+'_').setResultsName("op")

    overlay_spec = pp.nestedExpr(opener='(',
                                 closer=')',
                                 ignoreExpr=None
                             ).setResultsName("spec")

    value = pp.Word(pp.alphas+pp.nums+'_').setResultsName("value")

    op_settings = pp.nestedExpr(opener='[',
                                closer=']',
                                ignoreExpr=None
                            ).setResultsName("op_settings")

    compositor_spec = pp.OneOrMore(pp.Group(mode + op + overlay_spec + value
                                            + pp.Optional(op_settings)))

    @classmethod
    def parse(cls, line, ns={}):
        """
        Parse compositor specifications, returning a list Compositors
        """
        definitions = []
        parses  = [p for p in cls.compositor_spec.scanString(line)]
        if len(parses) != 1:
            raise SyntaxError("Invalid specification syntax.")
        else:
            e = parses[0][2]
            processed = line[:e]
            if (processed.strip() != line.strip()):
                raise SyntaxError("Failed to parse remainder of string: %r" % line[e:])

        opmap = {op.__name__:op for op in Compositor.operations}
        for group in cls.compositor_spec.parseString(line):

            if ('mode' not in group) or group['mode'] not in ['data', 'display']:
                raise SyntaxError("Either data or display mode must be specified.")
            mode = group['mode']

            kwargs = {}
            operation = opmap[group['op']]
            spec = ' '.join(group['spec'].asList()[0])

            if  group['op'] not in opmap:
                raise SyntaxError("Operation %s not available for use with compositors."
                                  % group['op'])
            if  'op_settings' in group:
                kwargs = cls.todict(group['op_settings'][0], 'brackets', ns=ns)

            definition = Compositor(str(spec), operation, str(group['value']), mode, **kwargs)
            definitions.append(definition)
        return definitions
<|endoftext|>


<commit_before>"""Wrap the PBS script with another script that captures stdout and
stderr, and provides support for checking the status of the job.
"""

import shlex
import pipes
from collections import namedtuple
import os.path
from textwrap import dedent
import os
import stat

PBSScriptParserResult = namedtuple(
    'PBSScriptParserResult',
    ['shebang',
     'directives',
     'script'])

class PBSScriptParser(object):
    def __init__(self):
        # the shebang
        #
        # do not set the default to a string: this is an important
        # part of the script and if we make a bad assumption then
        # wierd and hard-to-track-down errors will likely crop up.
        # instead, better to fail upfront if no shebang is provided.
        self._shebang = None

        # directives is a list of the qsub directives embedded in the script:
        # eg:
        #  #PBS -l nodes=1:ppn=1
        self._directives = list()

        # these are the rest of the contents of the script
        self._lines = list()

        # position
        self._line_counter = 1

    def is_shebang(self, line):
        return self._line_counter == 1 and line.startswith('#!')

    def handle_shebang(self, line):
        self._shebang = line.strip()

    def handle_directive(self, line, prefix='#PBS'):
        assert line.startswith(prefix)
        stripped = line.lstrip(prefix)
        directives = shlex.split(stripped)

        if not self._directives:
            self._directives.append(prefix)
        self._directives.extend(directives)

    def is_directive(self, line):
        return line.startswith('#PBS')

    def handle_script(self, line):
        self._lines.append(line)

    def result(self):
        # see comment in __init__ for why
        assert self._shebang is not None
        script = ''.join(self._lines)
        return PBSScriptParserResult(shebang=self._shebang,
                                     directives=self._directives,
                                     script=script)

    def _parse_line(self, line):
        if self.is_shebang(line):
            self.handle_shebang(line)
        elif self.is_directive(line):
            self.handle_directive(line)
        else:
            self.handle_script(line)

        self._line_counter += 1

    def _parse_lines(self, line_itr):
        for line in line_itr:
            self._parse_line(line)

        return self.result()

    def _parse_file(self, fd):
        return self._parse_lines(fd)

    def _parse_path(self, path):
        with open(path, 'r') as fd:
            return self._parse_file(fd)

    def parses(self, script_string):
        return self._parse_lines(script_string.split('\n'))

    def parse(self, path_or_file):
        if isinstance(path_or_file, str):
            return self._parse_path(path_or_file)
        elif isinstance(path_or_file, file):
            return self._parse_file(path_or_file)
        else:
            raise ValueError('Unsupport type {}'.format(path_or_file))

class Script(object):
    def __init__(self, name, contents,
                 mode=stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH):
        self._name = name
        self._contents = contents
        self._mode = mode

    @property
    def name(self):
        "Name (or path) of the script"
        return self._name

    @property
    def script(self):
        "The script body"
        return self._contents

    @property
    def mode(self):
        "The script execution bits (passed to `os.chmod`"
        return self._mode

    @mode.setter
    def mode(self, bits):
        self._mode = bits

    def write(self, directory):
        "Writes the script into a file name ``name`` in the directory"
        with open(os.path.join(directory, self.name), 'w') as fd:
            fd.write(self.script)
            os.chmod(fd.name, self.mode)

class Status(object):
    "The various states of a job"

    started = 'started'
    success = 'success'
    failure = 'failure'

class WrappedScript(object):
    def __init__(self, entrypoint, wrapped,
                 status='STATUS.txt',
                 stdout='STDOUT.txt',
                 stderr='STDERR.txt'):
        self._entrypoint = entrypoint
        self._wrapped = wrapped
        self.status = status
        self.stdout = stdout
        self.stderr = stderr

    @property
    def entrypoint(self):
        "The :class:`Script` that is the main entrypoint (the wrapper)"
        return self._entrypoint

    @property
    def wrapped(self):
        "The wrapped script"
        return self._wrapped

class Wrapper(object):
    def __init__(self, bash_location='/bin/bash'):
        # sometimes /usr/bin may not exist on grids
        self.shebang = '#!' + bash_location
        self.stdout = 'STDOUT.txt'
        self.stderr = 'STDERR.txt'
        self.status = 'STATUS.txt'

    def wrap(self, path_to_script, arguments=None):
        name = os.path.basename(path_to_script)
        name_wrapped = 'wrapped-{}'.format(name)

        args = arguments or list()

        parser = PBSScriptParser()
        tokens = parser.parse(path_to_script)

        wrapped_script = tokens.shebang + '\n' + tokens.script
        wrapped = Script(name_wrapped, wrapped_script)

        directives = ' '.join(tokens.directives)
        newscript_contents = dedent("""\
        {shebang}
        {directives}

        echo {state_started} > {state}
        ./wrapped-{old_name} {args} >{stdout} 2>{stderr}
        exit_code=$?

        if [ $exit_code -eq 0 ];then
            echo {state_success} > {state}
        else
            echo {state_failure} > {state}
        fi
        """.format(shebang=self.shebang,
                   directives=directives,
                   old_name=name,
                   args=' '.join(map(pipes.quote, args)),
                   state=self.status,
                   stdout=self.stdout,
                   stderr=self.stderr,
                   state_started=Status.started,
                   state_success=Status.success,
                   state_failure=Status.failure
                   ))

        newscript = Script(name, newscript_contents)

        return WrappedScript(newscript, wrapped,
                             status=self.status,
                             stdout=self.stdout,
                             stderr=self.stderr)
<commit_msg>add method to write wrapped script to a directory<commit_after>"""Wrap the PBS script with another script that captures stdout and
stderr, and provides support for checking the status of the job.
"""

import shlex
import pipes
from collections import namedtuple
import os.path
from textwrap import dedent
import os
import stat

PBSScriptParserResult = namedtuple(
    'PBSScriptParserResult',
    ['shebang',
     'directives',
     'script'])

class PBSScriptParser(object):
    def __init__(self):
        # the shebang
        #
        # do not set the default to a string: this is an important
        # part of the script and if we make a bad assumption then
        # wierd and hard-to-track-down errors will likely crop up.
        # instead, better to fail upfront if no shebang is provided.
        self._shebang = None

        # directives is a list of the qsub directives embedded in the script:
        # eg:
        #  #PBS -l nodes=1:ppn=1
        self._directives = list()

        # these are the rest of the contents of the script
        self._lines = list()

        # position
        self._line_counter = 1

    def is_shebang(self, line):
        return self._line_counter == 1 and line.startswith('#!')

    def handle_shebang(self, line):
        self._shebang = line.strip()

    def handle_directive(self, line, prefix='#PBS'):
        assert line.startswith(prefix)
        stripped = line.lstrip(prefix)
        directives = shlex.split(stripped)

        if not self._directives:
            self._directives.append(prefix)
        self._directives.extend(directives)

    def is_directive(self, line):
        return line.startswith('#PBS')

    def handle_script(self, line):
        self._lines.append(line)

    def result(self):
        # see comment in __init__ for why
        assert self._shebang is not None
        script = ''.join(self._lines)
        return PBSScriptParserResult(shebang=self._shebang,
                                     directives=self._directives,
                                     script=script)

    def _parse_line(self, line):
        if self.is_shebang(line):
            self.handle_shebang(line)
        elif self.is_directive(line):
            self.handle_directive(line)
        else:
            self.handle_script(line)

        self._line_counter += 1

    def _parse_lines(self, line_itr):
        for line in line_itr:
            self._parse_line(line)

        return self.result()

    def _parse_file(self, fd):
        return self._parse_lines(fd)

    def _parse_path(self, path):
        with open(path, 'r') as fd:
            return self._parse_file(fd)

    def parses(self, script_string):
        return self._parse_lines(script_string.split('\n'))

    def parse(self, path_or_file):
        if isinstance(path_or_file, str):
            return self._parse_path(path_or_file)
        elif isinstance(path_or_file, file):
            return self._parse_file(path_or_file)
        else:
            raise ValueError('Unsupport type {}'.format(path_or_file))

class Script(object):
    def __init__(self, name, contents,
                 mode=stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH):
        self._name = name
        self._contents = contents
        self._mode = mode

    @property
    def name(self):
        "Name (or path) of the script"
        return self._name

    @property
    def script(self):
        "The script body"
        return self._contents

    @property
    def mode(self):
        "The script execution bits (passed to `os.chmod`"
        return self._mode

    @mode.setter
    def mode(self, bits):
        self._mode = bits

    def write(self, directory):
        "Writes the script into a file name ``name`` in the directory"
        with open(os.path.join(directory, self.name), 'w') as fd:
            fd.write(self.script)
            os.chmod(fd.name, self.mode)

class Status(object):
    "The various states of a job"

    started = 'started'
    success = 'success'
    failure = 'failure'

class WrappedScript(object):
    def __init__(self, entrypoint, wrapped,
                 status='STATUS.txt',
                 stdout='STDOUT.txt',
                 stderr='STDERR.txt'):
        self._entrypoint = entrypoint
        self._wrapped = wrapped
        self.status = status
        self.stdout = stdout
        self.stderr = stderr

    @property
    def entrypoint(self):
        "The :class:`Script` that is the main entrypoint (the wrapper)"
        return self._entrypoint

    @property
    def wrapped(self):
        "The wrapped script"
        return self._wrapped

    def write(self, directory):
        self.entrypoint.write(directory)
        self.wrapped.write(directory)
        return [os.path.join(directory, name)
                for name in [self.entrypoint.name,
                             self.wrapped.name]]

class Wrapper(object):
    def __init__(self, bash_location='/bin/bash'):
        # sometimes /usr/bin may not exist on grids
        self.shebang = '#!' + bash_location
        self.stdout = 'STDOUT.txt'
        self.stderr = 'STDERR.txt'
        self.status = 'STATUS.txt'

    def wrap(self, path_to_script, arguments=None):
        name = os.path.basename(path_to_script)
        name_wrapped = 'wrapped-{}'.format(name)

        args = arguments or list()

        parser = PBSScriptParser()
        tokens = parser.parse(path_to_script)

        wrapped_script = tokens.shebang + '\n' + tokens.script
        wrapped = Script(name_wrapped, wrapped_script)

        directives = ' '.join(tokens.directives)
        newscript_contents = dedent("""\
        {shebang}
        {directives}

        echo {state_started} > {state}
        ./wrapped-{old_name} {args} >{stdout} 2>{stderr}
        exit_code=$?

        if [ $exit_code -eq 0 ];then
            echo {state_success} > {state}
        else
            echo {state_failure} > {state}
        fi
        """.format(shebang=self.shebang,
                   directives=directives,
                   old_name=name,
                   args=' '.join(map(pipes.quote, args)),
                   state=self.status,
                   stdout=self.stdout,
                   stderr=self.stderr,
                   state_started=Status.started,
                   state_success=Status.success,
                   state_failure=Status.failure
                   ))

        newscript = Script(name, newscript_contents)

        return WrappedScript(newscript, wrapped,
                             status=self.status,
                             stdout=self.stdout,
                             stderr=self.stderr)
<|endoftext|>


<commit_before># wwwhisper - web access control.
# Copyright (C) 2012 Jan Wrobel <wrr@mixedbit.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Views that handle user authentication and authorization."""

from django.conf import settings
from django.contrib import auth
from django.core.context_processors import csrf
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.generic import View
from functools import wraps
from wwwhisper_auth import http
from wwwhisper_auth import url_path
from wwwhisper_auth.backend import AssertionVerificationException

import logging

logger = logging.getLogger(__name__)

class Auth(View):
    """Handles auth request from the HTTP server.

    Auth request determines whether a user is authorized to access a
    given location. It must be sent by the HTTP server for each
    request to wwwhisper protected location. Auth request includes all
    headers of the original request and a path the original request is
    trying to access. The result of the request determines the action
    to be performed by the HTTP server:

      401: The user is not authenticated (no valid session cookie set).
           and should be presented with a login page.

      403: The user is authenticated (the request contains a valid
           session cookie) but is not authorized to access the
           location. The error should be passed to the user. The
           'User' header in the returned response containts email of
           the user.

      400: Request is malformed (suspicious path format, 'User' header
           set in the request, ...).

      200: User is authenticated and authorized to access the location
           or the location does not require authorization. The
           original request should be allowed. The 'User' header in
           the returned response containts email of the user.

      Any other result code should be passed to the user without
      granting access.

      Auth view does not need to be externally accessible.
    """

    locations_collection = None

    @method_decorator(http.never_ever_cache)
    def get(self, request):
        """Invoked by the HTTP server with a single path argument.

        The HTTP server should pass the path argument verbatim,
        without any transformations or decoding. Access control
        mechanism should work on user visible paths, not paths after
        internal rewrites performed by the server.

        At the moment, the path is allowed to contain a query part,
        which is ignored (this is because nginx does not expose
        encoded path without the query part).

        The method follows be conservative in what you accept
        principle. The path should be absolute and normalized, without
        fragment id, otherwise access is denied. Browsers in normal
        operations perform path normalization and do not send fragment
        id. Multiple consecutive '/' separators are permitted, because
        these are not normalized by browsers, and are used by
        legitimate applications.  Paths with '/./' and '/../', should
        not be normally sent by browsers and can be a sign of
        something suspicious happening. It is extremely important that
        wwwhisper does not perform any path transformations that are
        not be compatible with transformations done by the HTTP
        server.
       """
        encoded_path = self._extract_encoded_path_argument(request)
        if encoded_path is None:
            return http.HttpResponseBadRequest(
                "Auth request should have 'path' argument.")

        # Do not allow requests that contain the 'User' header. The
        # header is passed to backends and must be guaranteed to be
        # set by wwwhisper.
        # This check should already be performed by HTTP server.
        if request.META.has_key('HTTP_USER'):
            return http.HttpResponseBadRequest(
                "Client can not set the 'User' header")

        debug_msg = "Auth request to '%s'" % (encoded_path)

        path_validation_error = None
        if url_path.contains_fragment(encoded_path):
            path_validation_error = "Path should not include fragment ('#')"
        else:
            stripped_path = url_path.strip_query(encoded_path)
            decoded_path = url_path.decode(stripped_path)
            decoded_path = url_path.collapse_slashes(decoded_path)
            if not url_path.is_canonical(decoded_path):
                path_validation_error = 'Path should be absolute and ' \
                    'normalized (starting with / without /../ or /./ or //).'
        if path_validation_error is not None:
            logger.debug('%s: incorrect path.' % (debug_msg))
            return http.HttpResponseBadRequest(path_validation_error)

        user = request.user
        location = self.locations_collection.find_location(
            request.site_id, decoded_path)

        if (user and user.is_authenticated() and
            user.get_profile().site_id == request.site_id):
            debug_msg += " by '%s'" % (user.email)
            respone = None

            if location is not None and location.can_access(user.uuid):
                logger.debug('%s: access granted.' % (debug_msg))
                response =  http.HttpResponseOK('Access granted.')
            else:
                logger.debug('%s: access denied.' % (debug_msg))
                response = http.HttpResponseNotAuthorized()
            response['User'] = user.email
            return response

        if (location is not None and location.open_access_granted() and
            not location.open_access_requires_login()):
            logger.debug('%s: authentication not required, access granted.'
                         % (debug_msg))
            return http.HttpResponseOK('Access granted.')
        logger.debug('%s: user not authenticated.' % (debug_msg))
        return http.HttpResponseNotAuthenticated()

    @staticmethod
    def _extract_encoded_path_argument(request):
        """Get 'path' argument or None.

        Standard Django mechanism for accessing arguments is not used
        because path is needed in a raw, encoded form. Django would
        decode it, making it impossible to correctly recognize the
        query part and to determine if the path contains fragment.
        """
        request_path_and_args = request.get_full_path()
        assert request_path_and_args.startswith(request.path)
        args = request_path_and_args[len(request.path):]
        if not args.startswith('?path='):
            return None
        return args[len('?path='):]

def csrf_cookie_http_only(decorated_function):
    """Enables 'httponly' flag for CSRF protection cookie.

    The flag protects the cookie from javascript access.
    """
    @wraps(decorated_function)
    def wrapper(*args, **kwargs):
        response = decorated_function(*args, **kwargs)
        if response.cookies.has_key(settings.CSRF_COOKIE_NAME):
            response.cookies[settings.CSRF_COOKIE_NAME]['httponly'] = True
        return response
    return wrapper

class CsrfToken(View):
    """Establishes Cross Site Request Forgery protection token."""

    # TODO: Prevent CSRF protection cookie expiration.
    @method_decorator(http.never_ever_cache)
    @method_decorator(csrf_cookie_http_only)
    @method_decorator(ensure_csrf_cookie)
    def post(self, request):
        """Returns CSRF protection token in a cookie and a response body.

        The method must be called before any CSRF protected HTTP
        method is called (all HTTP methods of views that extend
        RestView). Returned token must be set in 'X-CSRFToken' header
        when the protected method is called, otherwise the call
        fails. It is enough to get the token once and reuse it for all
        subsequent calls to CSRF protected methods.

        POST is used instead of GET as an extra precaution against
        token leakage to sites from different origin. The token could
        possibly leak if a browser interpreted returned json as
        script, css or image and for example invoked js error handler
        with part of the response body .
        """
        csrf_token = csrf(request).values()[0]
        return http.HttpResponseOKJson({'csrfToken': str(csrf_token)})

class Login(http.RestView):
    """Allows a user to authenticates with BrowserID."""

    def post(self, request, assertion):
        """Logs a user in (establishes a session cookie).

        Verifies BrowserID assertion and check that a user with an
        email verified by the BrowserID is known (added to users
        list)."""
        if assertion == None:
            return http.HttpResponseBadRequest('BrowserId assertion not set.')
        try:
            user = auth.authenticate(site_id=request.site_id,
                                     site_url=request.site_url,
                                     assertion=assertion)
        except AssertionVerificationException as ex:
            return http.HttpResponseBadRequest(str(ex))
        if user is not None:
            auth.login(request, user)
            logger.debug('%s successfully logged.' % (user.email))
            return http.HttpResponseNoContent()
        else:
            # Unkown user.
            # Return not authorized because request was well formed (400
            # doesn't seem appropriate).
            return http.HttpResponseNotAuthorized()

class Logout(http.RestView):
    """Allows a user to logout."""

    def post(self, request):
        """Logs a user out (invalidates a session cookie)."""
        auth.logout(request)
        return http.HttpResponseNoContent()

class WhoAmI(http.RestView):
    """Allows to obtain an email of a currently logged in user."""

    def get(self, request):
        """Returns an email or an authentication required error."""
        user = request.user
        if (user and user.is_authenticated() and
            user.get_profile().site_id == request.site_id):
            return http.HttpResponseOKJson({'email': user.email})
        return http.HttpResponseNotAuthenticated()
<commit_msg>Debug when log assertion verification fails.<commit_after># wwwhisper - web access control.
# Copyright (C) 2012 Jan Wrobel <wrr@mixedbit.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Views that handle user authentication and authorization."""

from django.conf import settings
from django.contrib import auth
from django.core.context_processors import csrf
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.generic import View
from functools import wraps
from wwwhisper_auth import http
from wwwhisper_auth import url_path
from wwwhisper_auth.backend import AssertionVerificationException

import logging

logger = logging.getLogger(__name__)

class Auth(View):
    """Handles auth request from the HTTP server.

    Auth request determines whether a user is authorized to access a
    given location. It must be sent by the HTTP server for each
    request to wwwhisper protected location. Auth request includes all
    headers of the original request and a path the original request is
    trying to access. The result of the request determines the action
    to be performed by the HTTP server:

      401: The user is not authenticated (no valid session cookie set).
           and should be presented with a login page.

      403: The user is authenticated (the request contains a valid
           session cookie) but is not authorized to access the
           location. The error should be passed to the user. The
           'User' header in the returned response containts email of
           the user.

      400: Request is malformed (suspicious path format, 'User' header
           set in the request, ...).

      200: User is authenticated and authorized to access the location
           or the location does not require authorization. The
           original request should be allowed. The 'User' header in
           the returned response containts email of the user.

      Any other result code should be passed to the user without
      granting access.

      Auth view does not need to be externally accessible.
    """

    locations_collection = None

    @method_decorator(http.never_ever_cache)
    def get(self, request):
        """Invoked by the HTTP server with a single path argument.

        The HTTP server should pass the path argument verbatim,
        without any transformations or decoding. Access control
        mechanism should work on user visible paths, not paths after
        internal rewrites performed by the server.

        At the moment, the path is allowed to contain a query part,
        which is ignored (this is because nginx does not expose
        encoded path without the query part).

        The method follows be conservative in what you accept
        principle. The path should be absolute and normalized, without
        fragment id, otherwise access is denied. Browsers in normal
        operations perform path normalization and do not send fragment
        id. Multiple consecutive '/' separators are permitted, because
        these are not normalized by browsers, and are used by
        legitimate applications.  Paths with '/./' and '/../', should
        not be normally sent by browsers and can be a sign of
        something suspicious happening. It is extremely important that
        wwwhisper does not perform any path transformations that are
        not be compatible with transformations done by the HTTP
        server.
       """
        encoded_path = self._extract_encoded_path_argument(request)
        if encoded_path is None:
            return http.HttpResponseBadRequest(
                "Auth request should have 'path' argument.")

        # Do not allow requests that contain the 'User' header. The
        # header is passed to backends and must be guaranteed to be
        # set by wwwhisper.
        # This check should already be performed by HTTP server.
        if request.META.has_key('HTTP_USER'):
            return http.HttpResponseBadRequest(
                "Client can not set the 'User' header")

        debug_msg = "Auth request to '%s'" % (encoded_path)

        path_validation_error = None
        if url_path.contains_fragment(encoded_path):
            path_validation_error = "Path should not include fragment ('#')"
        else:
            stripped_path = url_path.strip_query(encoded_path)
            decoded_path = url_path.decode(stripped_path)
            decoded_path = url_path.collapse_slashes(decoded_path)
            if not url_path.is_canonical(decoded_path):
                path_validation_error = 'Path should be absolute and ' \
                    'normalized (starting with / without /../ or /./ or //).'
        if path_validation_error is not None:
            logger.debug('%s: incorrect path.' % (debug_msg))
            return http.HttpResponseBadRequest(path_validation_error)

        user = request.user
        location = self.locations_collection.find_location(
            request.site_id, decoded_path)

        # Makes sure user is authenticated and belongs to the current
        # site (auth backend just ensures user exists).
        if (user and user.is_authenticated() and
            user.get_profile().site_id == request.site_id):

            debug_msg += " by '%s'" % (user.email)
            respone = None

            if location is not None and location.can_access(user.uuid):
                logger.debug('%s: access granted.' % (debug_msg))
                response =  http.HttpResponseOK('Access granted.')
            else:
                logger.debug('%s: access denied.' % (debug_msg))
                response = http.HttpResponseNotAuthorized()
            response['User'] = user.email
            return response

        if (location is not None and location.open_access_granted() and
            not location.open_access_requires_login()):
            logger.debug('%s: authentication not required, access granted.'
                         % (debug_msg))
            return http.HttpResponseOK('Access granted.')
        logger.debug('%s: user not authenticated.' % (debug_msg))
        return http.HttpResponseNotAuthenticated()

    @staticmethod
    def _extract_encoded_path_argument(request):
        """Get 'path' argument or None.

        Standard Django mechanism for accessing arguments is not used
        because path is needed in a raw, encoded form. Django would
        decode it, making it impossible to correctly recognize the
        query part and to determine if the path contains fragment.
        """
        request_path_and_args = request.get_full_path()
        assert request_path_and_args.startswith(request.path)
        args = request_path_and_args[len(request.path):]
        if not args.startswith('?path='):
            return None
        return args[len('?path='):]

def csrf_cookie_http_only(decorated_function):
    """Enables 'httponly' flag for CSRF protection cookie.

    The flag protects the cookie from javascript access.
    """
    @wraps(decorated_function)
    def wrapper(*args, **kwargs):
        response = decorated_function(*args, **kwargs)
        if response.cookies.has_key(settings.CSRF_COOKIE_NAME):
            response.cookies[settings.CSRF_COOKIE_NAME]['httponly'] = True
        return response
    return wrapper

class CsrfToken(View):
    """Establishes Cross Site Request Forgery protection token."""

    # TODO: Prevent CSRF protection cookie expiration.
    @method_decorator(http.never_ever_cache)
    @method_decorator(csrf_cookie_http_only)
    @method_decorator(ensure_csrf_cookie)
    def post(self, request):
        """Returns CSRF protection token in a cookie and a response body.

        The method must be called before any CSRF protected HTTP
        method is called (all HTTP methods of views that extend
        RestView). Returned token must be set in 'X-CSRFToken' header
        when the protected method is called, otherwise the call
        fails. It is enough to get the token once and reuse it for all
        subsequent calls to CSRF protected methods.

        POST is used instead of GET as an extra precaution against
        token leakage to sites from different origin. The token could
        possibly leak if a browser interpreted returned json as
        script, css or image and for example invoked js error handler
        with part of the response body .
        """
        csrf_token = csrf(request).values()[0]
        return http.HttpResponseOKJson({'csrfToken': str(csrf_token)})

class Login(http.RestView):
    """Allows a user to authenticates with BrowserID."""

    def post(self, request, assertion):
        """Logs a user in (establishes a session cookie).

        Verifies BrowserID assertion and check that a user with an
        email verified by the BrowserID is known (added to users
        list)."""
        if assertion == None:
            return http.HttpResponseBadRequest('BrowserId assertion not set.')
        try:
            user = auth.authenticate(site_id=request.site_id,
                                     site_url=request.site_url,
                                     assertion=assertion)
        except AssertionVerificationException as ex:
            logger.debug('Assertion verification failed.')
            return http.HttpResponseBadRequest(str(ex))
        if user is not None:
            auth.login(request, user)
            logger.debug('%s successfully logged.' % (user.email))
            return http.HttpResponseNoContent()
        else:
            # Unkown user.
            # Return not authorized because request was well formed (400
            # doesn't seem appropriate).
            return http.HttpResponseNotAuthorized()

class Logout(http.RestView):
    """Allows a user to logout."""

    def post(self, request):
        """Logs a user out (invalidates a session cookie)."""
        auth.logout(request)
        return http.HttpResponseNoContent()

class WhoAmI(http.RestView):
    """Allows to obtain an email of a currently logged in user."""

    def get(self, request):
        """Returns an email or an authentication required error."""
        user = request.user
        if (user and user.is_authenticated() and
            user.get_profile().site_id == request.site_id):
            return http.HttpResponseOKJson({'email': user.email})
        return http.HttpResponseNotAuthenticated()
<|endoftext|>


<commit_before>"""
Packages the files into a folder ready to upload to AWS.

This involves:

 (a) Copy all the files into the folder.
 (b) Run pip install -r requirements.txt -t /path/to/folder
 (c) Zip it all up
"""

import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile
from distutils import dir_util

import factoryfactory
import pip

from . import configuration

class Package(factoryfactory.Serviceable):
    """
    Creates a bundled package
    """

    def __init__(self, cfg, terraform=False):
        """
        @param cfg
            The FunctionConfig from which the package is to be built.
        @param base_dir
            The base directory relative to which paths are to be resolved.
        @param bundle_folder
            The temporary folder into which the packgage is to be created.
        """
        self.root = self.services.get(configuration.Configuration).root
        self.runtime = cfg.runtime
        self.terraform = terraform
        self.build = cfg.build
        self.build.resolve(self.root)
        self.test = cfg.test
        self.bundle_folder = cfg.build.bundle

    def copy_files(self):
        """
        Copies the files from the source folder into the bundle.
        """
        if os.path.exists(self.bundle_folder):
            shutil.rmtree(self.bundle_folder)
        shutil.copytree(
            self.build.source, self.bundle_folder,
            ignore=shutil.ignore_patterns(*self.build.ignore)
        )

    def install_requirement_file(self, requirement):
        """
        Installs the requirements specified in requirements.txt into the bundle.
        """
        stdout_redirect = sys.stderr if self.terraform else sys.stdout
        with open(requirement) as f, \
            tempfile.NamedTemporaryFile(mode='w+t') as t:
            for line in f:
                s = line.strip()
                s = re.sub(r'^-e\s+', '', s)
                t.file.write(s + os.linesep)
            t.flush()
            compile_args = [ '--compile' if self.build.compile_dependencies else '--no-compile' ]
            if self.build.use_docker:
                cmd = [
                    'docker', 'run',
                    '-v', os.path.realpath(t.name) + ':/requirements.txt',
                    '-v', os.path.realpath(self.bundle_folder) + ':/bundle',
                    '--rm', 'python:3.6.3',
                    'pip', 'install', '-r', '/requirements.txt', '-t', '/bundle'
                ]
            else:
                cmd = [ 'pip', 'install', '-r', t.name, '-t', self.bundle_folder ]
            subprocess.run(cmd + compile_args, stdout=stdout_redirect)

        #
        # pip doesn't preserve timestamps when installing files.
        # I think it's supposed to, but it doesn't seem to work.
        # Therefore we'll set the timestamps of all downloaded files to
        # the timestamp of the requirements.txt file.
        #
        times = (
            os.path.getatime(requirement),
            os.path.getmtime(requirement)
        )
        for dirname, subdirs, files in os.walk(self.bundle_folder):
            for filename in files + subdirs:
                filepath = os.path.join(dirname, filename)
                os.utime(filepath, times)

    def install_requirements(self):
        for requirement in self.build.requirements or []:
            self.install_requirement_file(requirement.file)

    def create_archive(self):
        """
        Creates the archive file.
        """
        dirname = os.path.dirname(self.build.package)
        os.makedirs(dirname, exist_ok=True)
        base_name, fmt = os.path.splitext(self.build.package)
        fmt = fmt.replace(os.path.extsep, '') or 'zip'
        shutil.make_archive(base_name, fmt, self.bundle_folder, './', True)

    def remove_bundle_folder(self):
        """
        Removes the bundle folder.
        """
        shutil.rmtree(self.bundle_folder)
        self.bundle_folder = None

    def create(self):
        """
        Performs all the above steps to create the bundle.
        """
        try:
            self.copy_files()
            self.install_requirements()
            self.create_archive()
        finally:
            if not self.test:
                self.remove_bundle_folder()

class BuildCommand(factoryfactory.Serviceable):

    def __init__(self, functions, terraform):
        self.functions = functions
        self.terraform = terraform

    def run(self):
        config = self.services.get(configuration.Configuration)
        functions = config.get_functions(self.functions)
        for name in functions:
            funcdef = functions[name]
            package = self.services.get(Package, funcdef, terraform=self.terraform)
            package.create()
<commit_msg>Remove redundant import.<commit_after>"""
Packages the files into a folder ready to upload to AWS.

This involves:

 (a) Copy all the files into the folder.
 (b) Run pip install -r requirements.txt -t /path/to/folder
 (c) Zip it all up
"""

import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile

import factoryfactory
import pip

from . import configuration

class Package(factoryfactory.Serviceable):
    """
    Creates a bundled package
    """

    def __init__(self, cfg, terraform=False):
        """
        @param cfg
            The FunctionConfig from which the package is to be built.
        @param base_dir
            The base directory relative to which paths are to be resolved.
        @param bundle_folder
            The temporary folder into which the packgage is to be created.
        """
        self.root = self.services.get(configuration.Configuration).root
        self.runtime = cfg.runtime
        self.terraform = terraform
        self.build = cfg.build
        self.build.resolve(self.root)
        self.test = cfg.test
        self.bundle_folder = cfg.build.bundle

    def copy_files(self):
        """
        Copies the files from the source folder into the bundle.
        """
        if os.path.exists(self.bundle_folder):
            shutil.rmtree(self.bundle_folder)
        shutil.copytree(
            self.build.source, self.bundle_folder,
            ignore=shutil.ignore_patterns(*self.build.ignore)
        )

    def install_requirement_file(self, requirement):
        """
        Installs the requirements specified in requirements.txt into the bundle.
        """
        stdout_redirect = sys.stderr if self.terraform else sys.stdout
        with open(requirement) as f, \
            tempfile.NamedTemporaryFile(mode='w+t') as t:
            for line in f:
                s = line.strip()
                s = re.sub(r'^-e\s+', '', s)
                t.file.write(s + os.linesep)
            t.flush()
            compile_args = [ '--compile' if self.build.compile_dependencies else '--no-compile' ]
            if self.build.use_docker:
                cmd = [
                    'docker', 'run',
                    '-v', os.path.realpath(t.name) + ':/requirements.txt',
                    '-v', os.path.realpath(self.bundle_folder) + ':/bundle',
                    '--rm', 'python:3.6.3',
                    'pip', 'install', '-r', '/requirements.txt', '-t', '/bundle'
                ]
            else:
                cmd = [ 'pip', 'install', '-r', t.name, '-t', self.bundle_folder ]
            subprocess.run(cmd + compile_args, stdout=stdout_redirect)

        #
        # pip doesn't preserve timestamps when installing files.
        # I think it's supposed to, but it doesn't seem to work.
        # Therefore we'll set the timestamps of all downloaded files to
        # the timestamp of the requirements.txt file.
        #
        times = (
            os.path.getatime(requirement),
            os.path.getmtime(requirement)
        )
        for dirname, subdirs, files in os.walk(self.bundle_folder):
            for filename in files + subdirs:
                filepath = os.path.join(dirname, filename)
                os.utime(filepath, times)

    def install_requirements(self):
        for requirement in self.build.requirements or []:
            self.install_requirement_file(requirement.file)

    def create_archive(self):
        """
        Creates the archive file.
        """
        dirname = os.path.dirname(self.build.package)
        os.makedirs(dirname, exist_ok=True)
        base_name, fmt = os.path.splitext(self.build.package)
        fmt = fmt.replace(os.path.extsep, '') or 'zip'
        shutil.make_archive(base_name, fmt, self.bundle_folder, './', True)

    def remove_bundle_folder(self):
        """
        Removes the bundle folder.
        """
        shutil.rmtree(self.bundle_folder)
        self.bundle_folder = None

    def create(self):
        """
        Performs all the above steps to create the bundle.
        """
        try:
            self.copy_files()
            self.install_requirements()
            self.create_archive()
        finally:
            if not self.test:
                self.remove_bundle_folder()

class BuildCommand(factoryfactory.Serviceable):

    def __init__(self, functions, terraform):
        self.functions = functions
        self.terraform = terraform

    def run(self):
        config = self.services.get(configuration.Configuration)
        functions = config.get_functions(self.functions)
        for name in functions:
            funcdef = functions[name]
            package = self.services.get(Package, funcdef, terraform=self.terraform)
            package.create()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2015-2021 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Version information for Invenio-OAuthClient.

This file is imported by ``invenio_oauthclient.__init__``,
and parsed by ``setup.py``.
"""

__version__ = '1.5.4'
<commit_msg>release: v2.0.0.dev0<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2015-2021 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Version information for Invenio-OAuthClient.

This file is imported by ``invenio_oauthclient.__init__``,
and parsed by ``setup.py``.
"""

__version__ = '2.0.0.dev0'
<|endoftext|>


<commit_before>from globals import DATA_DIR

import logging
import os

def get_home_directory():
	if 'HOME' in os.environ:
		return os.environ['HOME']
	elif 'USERPROFILE' in os.environ:
		return os.environ['USERPROFILE']
	else:
		raise Exception('No home directory could be found.')

def has_reactor3():
	_config_directory = os.path.join(get_home_directory(),'.config','reactor-3')
	_worlds_directory = os.path.join(_config_directory, 'worlds')

	try:
		os.mkdir(_config_directory)
		logging.info('Created config directory: %s' % _config_directory)
	except OSError:
		pass

	try:
		os.mkdir(_worlds_directory)
		logging.info('Created worlds directory: %s' % _worlds_directory)
		return (_config_directory, _worlds_directory)
	except OSError:
		return (_config_directory, _worlds_directory)

def get_maps():
	_map_dir = os.path.join(DATA_DIR, 'maps')
	_maps = []

	for (dirpath, dirname, filenames) in os.walk(_map_dir):
		_maps.extend(filenames)

	return _maps

def get_worlds():
	_config_directory, _worlds_directory = has_reactor3()

	_dirs = []
	for (dirpath, dirname, filenames) in os.walk(_worlds_directory):
		_dirs.extend(dirname)
		break

	return _dirs

def create_world():
	_config_directory, _worlds_directory = has_reactor3()

	_world_name = str(len(get_worlds())+1)
	_world_directory = os.path.join(_worlds_directory, _world_name)

	try:
		os.mkdir(_world_directory)
		logging.info('Created world: %s' % _world_name)
		return _world_name
	except OSError:
		return False

def get_world(world):
	_config_directory, _worlds_directory = has_reactor3()
	_world_directory = os.path.join(_worlds_directory, str(world))

	return _world_directory<commit_msg>Profile path fix<commit_after>from globals import DATA_DIR

import logging
import os

def get_home_directory():
	return os.path.expanduser('~')

def has_reactor3():
	_config_directory = os.path.join(get_home_directory(),'.config','reactor-3')
	_worlds_directory = os.path.join(_config_directory, 'worlds')

	try:
		os.mkdir(_config_directory)
		logging.info('Created config directory: %s' % _config_directory)
	except OSError:
		pass

	try:
		os.mkdir(_worlds_directory)
		logging.info('Created worlds directory: %s' % _worlds_directory)
		return (_config_directory, _worlds_directory)
	except OSError:
		return (_config_directory, _worlds_directory)

def get_maps():
	_map_dir = os.path.join(DATA_DIR, 'maps')
	_maps = []

	for (dirpath, dirname, filenames) in os.walk(_map_dir):
		_maps.extend(filenames)

	return _maps

def get_worlds():
	_config_directory, _worlds_directory = has_reactor3()

	_dirs = []
	for (dirpath, dirname, filenames) in os.walk(_worlds_directory):
		_dirs.extend(dirname)
		break

	return _dirs

def create_world():
	_config_directory, _worlds_directory = has_reactor3()

	_world_name = str(len(get_worlds())+1)
	_world_directory = os.path.join(_worlds_directory, _world_name)

	try:
		os.mkdir(_world_directory)
		logging.info('Created world: %s' % _world_name)
		return _world_name
	except OSError:
		return False

def get_world(world):
	_config_directory, _worlds_directory = has_reactor3()
	_world_directory = os.path.join(_worlds_directory, str(world))

	return _world_directory<|endoftext|>


<commit_before>from matplotlib import pyplot as plt
import numpy as np

import param

from ..core import Dimension, ViewOperation, Overlay
from ..view import ItemTable, Matrix, VectorField, Contours

class chain(ViewOperation):
    """
    Definining a viewoperation chain is an easy way to define a new
    ViewOperation from a series of existing ones. The single argument
    is a callable that accepts an input view and returns a list of
    output views. To create the custom ViewOperation, you will need to
    supply this argument to a new instance of chain. For example:

    chain.instance(chain=lambda x: [cmap2rgb(operator(x).N, cmap='jet')])

    This is now a ViewOperation that sums the data in the input
    overlay and turns it into an RGB Matrix with the 'jet'
    colormap.
    """

    chain = param.Callable(doc="""A chain of existing ViewOperations.""")

    def _process(self, view, key=None):
        return self.p.chain(view)

class operator(ViewOperation):
    """
    Applies any arbitrary operator on the data (currently only
    supports Matrix views) and returns the result.
    """

    operator = param.Callable(np.add, doc="""
        The commutative operator to apply between the data attributes
        of the supplied Views. By default, performs elementwise
        addition across the Matrix arrays.""")

    unpack = param.Boolean(default=True, doc=""" Whether the operator
       is supplied the .data attributes as an unpack collection of
       arguments or as a list.""")

    label = param.String(default='Operation', doc="""
        The label for the result after having applied the operator.""")

    def _process(self, overlay, key=None):

        if not isinstance(overlay, Overlay):
            raise Exception("Operation requires an Overlay as input")

        if self.p.unpack:
            new_data = self.p.operator(*[el.data for el in overlay])
        else:
            new_data = self.p.operator([el.data for el in overlay])

        return [Matrix(new_data, bounds=overlay[0].bounds, label=self.p.label,
                            roi_bounds=overlay[0].roi_bounds)]

class convolve(ViewOperation):
    """
    Apply a convolution to an overlay using the top layer as the
    kernel used to convolve the bottom layer. Both input Matrix
    elements in the overlay should be single-channel.
    """

    label = param.String(default='Convolution', doc="""
        The label to identify the output of the Convolution.""")

    kernel_roi = param.NumericTuple(default=(0,0,0,0), length=4, doc="""
        A 2-dimensional slice of the kernel layer to use in the
        convolution in lbrt (left, bottom, right, top) format. By
        default, no slicing is applied.""")

    def _process(self, view, key=None):

        if len(view) != 2:
            raise Exception("Overlay of two layers required.")

        [target, kernel] = view[0], view[1]

        if len(target.data.shape) != 2:
            raise Exception("Convolve requires monochrome inputs.")

        xslice = slice(self.p.kernel_roi[0], self.p.kernel_roi[2])
        yslice = slice(self.p.kernel_roi[1], self.p.kernel_roi[3])

        k = kernel.data if self.p.kernel_roi == (0,0,0,0) else kernel[xslice, yslice].data

        fft1 = np.fft.fft2(target.data)
        fft2 = np.fft.fft2(k, s= target.data.shape)
        convolved_raw = np.fft.ifft2(fft1 * fft2).real

        k_rows, k_cols = k.shape
        rolled = np.roll(np.roll(convolved_raw, -(k_cols//2), axis=-1), -(k_rows//2), axis=-2)
        convolved = rolled / float(k.sum())

        return [Matrix(convolved, bounds=target.bounds)]

class contours(ViewOperation):
    """
    Given a Matrix with a single channel, annotate it with contour
    lines for a given set of contour levels.

    The return is a overlay with a Contours layer for each given
    level, overlaid on top of the input Matrix.
    """

    levels = param.NumericTuple(default=(0.5,), doc="""
         A list of scalar values used to specify the contour levels.""")

    label = param.String(default='Level', doc="""
      The label suffix used to label the resulting contour curves
      where the suffix is added to the label of the  input Matrix""")

    def _process(self, sheetview, key=None):

        figure_handle = plt.figure()
        (l, b, r, t) = sheetview.lbrt
        contour_set = plt.contour(sheetview.data, extent=(l, r, t, b),
                                  levels=self.p.levels)

        contours = []
        for level, cset in zip(self.p.levels, contour_set.collections):
            paths = cset.get_paths()
            lines = [path.vertices for path in paths]
            contours.append(Contours(lines, label=sheetview.label + ' ' + self.p.label))

        plt.close(figure_handle)

        if len(contours) == 1:
            return [(sheetview * contours[0])]
        else:
            return [sheetview * Overlay(contours)]

class vectorfield(ViewOperation):
    """
    Given a Matrix with a single channel, convert it to a
    VectorField object at a given spatial sampling interval. The
    values in the Matrix are assumed to correspond to the vector
    angle in radians and the value is assumed to be cyclic.

    If supplied with an overlay, the second sheetview in the overlay
    will be interpreted as the third vector dimension.
    """

    rows = param.Integer(default=10, doc="""
         Number of rows in the vector field.""")

    cols = param.Integer(default=10, doc="""
         Number of columns in the vector field.""")

    label = param.String(default='Vectors', doc="""
      The label suffix used to label the resulting vector field
      where the suffix is added to the label of the  input Matrix""")

    def _process(self, view, key=None):

        if isinstance(view, Overlay) and len(view) >= 2:
            radians, lengths = view[0], view[1]
        else:
            radians, lengths = view, None

        if not radians.value.cyclic:
            raise Exception("First input Matrix must be declared cyclic")

        l, b, r, t = radians.bounds.lbrt()
        X, Y = np.meshgrid(np.linspace(l, r, self.p.cols+2)[1:-1],
                           np.linspace(b, t, self.p.rows+2)[1:-1])

        vector_data = []
        for x, y in zip(X.flat, Y.flat):

            components = (x,y, radians[x,y])
            if lengths is not None:
                components += (lengths[x,y],)

            vector_data.append(components)

        value_dimension = Dimension('VectorField', cyclic=True, range=radians.range)
        return [VectorField(vector_data,
                            label=radians.label + ' ' + self.p.label,
                            value=value_dimension)]

class threshold(ViewOperation):
    """
    Threshold a given Matrix at a given level into the specified
    low and high values.  """

    level = param.Number(default=0.5, doc="""
       The value at which the threshold is applied. Values lower than
       the threshold map to the 'low' value and values above map to
       the 'high' value.""")

    high = param.Number(default=1.0, doc="""
      The value given to elements greater than (or equal to) the
      threshold.""")

    low = param.Number(default=0.0, doc="""
      The value given to elements below the threshold.""")

    label = param.String(default='Thresholded', doc="""
       The label suffix used to label the resulting sheetview where
       the suffix is added to the label of the input Matrix""")

    def _process(self, view, key=None):
        arr = view.data
        high = np.ones(arr.shape) * self.p.high
        low = np.ones(arr.shape) * self.p.low
        thresholded = np.where(arr > self.p.level, high, low)

        return [Matrix(thresholded,
                          label=view.label + ' ' + self.p.label)]

class roi_table(ViewOperation):
    """
    Compute a table of information from a Matrix within the
    indicated region-of-interest (ROI). The function applied must
    accept a numpy array and return either a single value or a
    dictionary of values which is returned as a ItemTable or ViewMap.

    The roi is specified using a boolean Matrix overlay (e.g as
    generated by threshold) where zero values are excluded from the
    ROI.
    """

    fn = param.Callable(default=np.mean, doc="""
        The function that is applied within the mask area to supply
        the relevant information. Other valid examples include np.sum,
        np.median, np.std etc.""")

    heading = param.String(default='result', doc="""
       If the output of the function is not a dictionary, this is the
       label given to the resulting value.""")

    label = param.String(default='ROI', doc="""
       The label suffix that labels the resulting table where this
       suffix is added to the label of the input data Matrix""")

    def _process(self, view, key=None):

        if not isinstance(view, Overlay) or len(view) != 2:
            raise Exception("A Overlay of two SheetViews is required.")

        mview, mask = view[0], view[1]

        if mview.data.shape != mask.data.shape:
            raise Exception("The data array shape of the mask layer"
                            " must match that of the data layer.")

        bit_mask = mask.data.astype(np.bool)
        roi = mview.data[bit_mask]

        results = self.p.fn(roi)
        if not isinstance(results, dict):
            results = {self.p.heading:results}

        return [ItemTable(results, label=mview.label + ' ' + self.p.label)]
<commit_msg>The roi_table operation now ensures some elements are always unmasked<commit_after>from matplotlib import pyplot as plt
import numpy as np

import param

from ..core import Dimension, ViewOperation, Overlay
from ..view import ItemTable, Matrix, VectorField, Contours

class chain(ViewOperation):
    """
    Definining a viewoperation chain is an easy way to define a new
    ViewOperation from a series of existing ones. The single argument
    is a callable that accepts an input view and returns a list of
    output views. To create the custom ViewOperation, you will need to
    supply this argument to a new instance of chain. For example:

    chain.instance(chain=lambda x: [cmap2rgb(operator(x).N, cmap='jet')])

    This is now a ViewOperation that sums the data in the input
    overlay and turns it into an RGB Matrix with the 'jet'
    colormap.
    """

    chain = param.Callable(doc="""A chain of existing ViewOperations.""")

    def _process(self, view, key=None):
        return self.p.chain(view)

class operator(ViewOperation):
    """
    Applies any arbitrary operator on the data (currently only
    supports Matrix views) and returns the result.
    """

    operator = param.Callable(np.add, doc="""
        The commutative operator to apply between the data attributes
        of the supplied Views. By default, performs elementwise
        addition across the Matrix arrays.""")

    unpack = param.Boolean(default=True, doc=""" Whether the operator
       is supplied the .data attributes as an unpack collection of
       arguments or as a list.""")

    label = param.String(default='Operation', doc="""
        The label for the result after having applied the operator.""")

    def _process(self, overlay, key=None):

        if not isinstance(overlay, Overlay):
            raise Exception("Operation requires an Overlay as input")

        if self.p.unpack:
            new_data = self.p.operator(*[el.data for el in overlay])
        else:
            new_data = self.p.operator([el.data for el in overlay])

        return [Matrix(new_data, bounds=overlay[0].bounds, label=self.p.label,
                            roi_bounds=overlay[0].roi_bounds)]

class convolve(ViewOperation):
    """
    Apply a convolution to an overlay using the top layer as the
    kernel used to convolve the bottom layer. Both input Matrix
    elements in the overlay should be single-channel.
    """

    label = param.String(default='Convolution', doc="""
        The label to identify the output of the Convolution.""")

    kernel_roi = param.NumericTuple(default=(0,0,0,0), length=4, doc="""
        A 2-dimensional slice of the kernel layer to use in the
        convolution in lbrt (left, bottom, right, top) format. By
        default, no slicing is applied.""")

    def _process(self, view, key=None):

        if len(view) != 2:
            raise Exception("Overlay of two layers required.")

        [target, kernel] = view[0], view[1]

        if len(target.data.shape) != 2:
            raise Exception("Convolve requires monochrome inputs.")

        xslice = slice(self.p.kernel_roi[0], self.p.kernel_roi[2])
        yslice = slice(self.p.kernel_roi[1], self.p.kernel_roi[3])

        k = kernel.data if self.p.kernel_roi == (0,0,0,0) else kernel[xslice, yslice].data

        fft1 = np.fft.fft2(target.data)
        fft2 = np.fft.fft2(k, s= target.data.shape)
        convolved_raw = np.fft.ifft2(fft1 * fft2).real

        k_rows, k_cols = k.shape
        rolled = np.roll(np.roll(convolved_raw, -(k_cols//2), axis=-1), -(k_rows//2), axis=-2)
        convolved = rolled / float(k.sum())

        return [Matrix(convolved, bounds=target.bounds)]

class contours(ViewOperation):
    """
    Given a Matrix with a single channel, annotate it with contour
    lines for a given set of contour levels.

    The return is a overlay with a Contours layer for each given
    level, overlaid on top of the input Matrix.
    """

    levels = param.NumericTuple(default=(0.5,), doc="""
         A list of scalar values used to specify the contour levels.""")

    label = param.String(default='Level', doc="""
      The label suffix used to label the resulting contour curves
      where the suffix is added to the label of the  input Matrix""")

    def _process(self, sheetview, key=None):

        figure_handle = plt.figure()
        (l, b, r, t) = sheetview.lbrt
        contour_set = plt.contour(sheetview.data, extent=(l, r, t, b),
                                  levels=self.p.levels)

        contours = []
        for level, cset in zip(self.p.levels, contour_set.collections):
            paths = cset.get_paths()
            lines = [path.vertices for path in paths]
            contours.append(Contours(lines, label=sheetview.label + ' ' + self.p.label))

        plt.close(figure_handle)

        if len(contours) == 1:
            return [(sheetview * contours[0])]
        else:
            return [sheetview * Overlay(contours)]

class vectorfield(ViewOperation):
    """
    Given a Matrix with a single channel, convert it to a
    VectorField object at a given spatial sampling interval. The
    values in the Matrix are assumed to correspond to the vector
    angle in radians and the value is assumed to be cyclic.

    If supplied with an overlay, the second sheetview in the overlay
    will be interpreted as the third vector dimension.
    """

    rows = param.Integer(default=10, doc="""
         Number of rows in the vector field.""")

    cols = param.Integer(default=10, doc="""
         Number of columns in the vector field.""")

    label = param.String(default='Vectors', doc="""
      The label suffix used to label the resulting vector field
      where the suffix is added to the label of the  input Matrix""")

    def _process(self, view, key=None):

        if isinstance(view, Overlay) and len(view) >= 2:
            radians, lengths = view[0], view[1]
        else:
            radians, lengths = view, None

        if not radians.value.cyclic:
            raise Exception("First input Matrix must be declared cyclic")

        l, b, r, t = radians.bounds.lbrt()
        X, Y = np.meshgrid(np.linspace(l, r, self.p.cols+2)[1:-1],
                           np.linspace(b, t, self.p.rows+2)[1:-1])

        vector_data = []
        for x, y in zip(X.flat, Y.flat):

            components = (x,y, radians[x,y])
            if lengths is not None:
                components += (lengths[x,y],)

            vector_data.append(components)

        value_dimension = Dimension('VectorField', cyclic=True, range=radians.range)
        return [VectorField(vector_data,
                            label=radians.label + ' ' + self.p.label,
                            value=value_dimension)]

class threshold(ViewOperation):
    """
    Threshold a given Matrix at a given level into the specified
    low and high values.  """

    level = param.Number(default=0.5, doc="""
       The value at which the threshold is applied. Values lower than
       the threshold map to the 'low' value and values above map to
       the 'high' value.""")

    high = param.Number(default=1.0, doc="""
      The value given to elements greater than (or equal to) the
      threshold.""")

    low = param.Number(default=0.0, doc="""
      The value given to elements below the threshold.""")

    label = param.String(default='Thresholded', doc="""
       The label suffix used to label the resulting sheetview where
       the suffix is added to the label of the input Matrix""")

    def _process(self, view, key=None):
        arr = view.data
        high = np.ones(arr.shape) * self.p.high
        low = np.ones(arr.shape) * self.p.low
        thresholded = np.where(arr > self.p.level, high, low)

        return [Matrix(thresholded,
                          label=view.label + ' ' + self.p.label)]

class roi_table(ViewOperation):
    """
    Compute a table of information from a Matrix within the
    indicated region-of-interest (ROI). The function applied must
    accept a numpy array and return either a single value or a
    dictionary of values which is returned as a ItemTable or ViewMap.

    The roi is specified using a boolean Matrix overlay (e.g as
    generated by threshold) where zero values are excluded from the
    ROI.
    """

    fn = param.Callable(default=np.mean, doc="""
        The function that is applied within the mask area to supply
        the relevant information. Other valid examples include np.sum,
        np.median, np.std etc.""")

    heading = param.String(default='result', doc="""
       If the output of the function is not a dictionary, this is the
       label given to the resulting value.""")

    label = param.String(default='ROI', doc="""
       The label suffix that labels the resulting table where this
       suffix is added to the label of the input data Matrix""")

    def _process(self, view, key=None):

        if not isinstance(view, Overlay) or len(view) != 2:
            raise Exception("A Overlay of two SheetViews is required.")

        mview, mask = view[0], view[1]

        if mview.data.shape != mask.data.shape:
            raise Exception("The data array shape of the mask layer"
                            " must match that of the data layer.")

        bit_mask = mask.data.astype(np.bool)
        roi = mview.data[bit_mask]

        if len(roi) == 0:
            raise Exception("No region of interest defined in ROI mask" )

        results = self.p.fn(roi)
        if not isinstance(results, dict):
            results = {self.p.heading:results}

        return [ItemTable(results, label=mview.label + ' ' + self.p.label)]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Tests of sparse matrix structures."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os

import numpy as np
from nose.tools import assert_raises

from ..sparse import csr_matrix

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

def setup():
    pass

def teardown():
    pass

#------------------------------------------------------------------------------
# Tests
#------------------------------------------------------------------------------

def test_sparse_csr():
    arr = np.zeros((4, 5))
    arr[0, 1] = 1
    arr[0, 2] = 2
    arr[1, 0] = 3
    arr[1, 3] = 4
    arr[3, 4] = 5

    assert_raises(NotImplementedError, csr_matrix, (arr,))
<commit_msg>WIP: tests.<commit_after># -*- coding: utf-8 -*-

"""Tests of sparse matrix structures."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os

import numpy as np
from nose.tools import assert_raises

from ..sparse import csr_matrix

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

def setup():
    pass

def teardown():
    pass

#------------------------------------------------------------------------------
# Tests
#------------------------------------------------------------------------------

def _dense_matrix_example():
    arr = np.zeros((4, 5))
    arr[0, 1] = 1
    arr[0, 2] = 2
    arr[1, 0] = 3
    arr[1, 3] = 4
    arr[3, 4] = 5
    return arr

def test_sparse_csr_check():
    arr = _dense_matrix_example()
    assert_raises(NotImplementedError, csr_matrix, (arr,))

    data_exp = np.arange(1, 6)

    data = data_exp
    channels = None
    spikes_ptr = None
    csr_matrix(data=data, channels=channels, spikes_ptr=spikes_ptr)
<|endoftext|>