<commit_before>from cached_property import cached_property
import sympy

from devito.ir.equations.algorithms import dimension_sort, lower_exprs
from devito.finite_differences.differentiable import diff2sympy
from devito.ir.support import (IterationSpace, DataSpace, Interval, IntervalGroup,
                               Stencil, detect_accesses, detect_oobs, detect_io,
                               build_intervals, build_iterators)
from devito.symbolics import CondEq
from devito.tools import Pickable, frozendict
from devito.types import Eq

__all__ = ['LoweredEq', 'ClusterizedEq', 'DummyEq']

class IREq(object):

    """
    A mixin providing operations common to all :mod:`ir` equation types.
    """
    _state = ('is_Increment', 'ispace', 'dspace', 'conditionals', 'implicit_dims')

    @property
    def is_Scalar(self):
        return self.lhs.is_Symbol

    is_scalar = is_Scalar

    @property
    def is_Tensor(self):
        return self.lhs.is_Indexed

    @property
    def is_Increment(self):
        return self._is_Increment

    @property
    def ispace(self):
        return self._ispace

    @property
    def dspace(self):
        return self._dspace

    @cached_property
    def dimensions(self):
        # Note: some dimensions may be in the iteration space but not in the
        # data space (e.g., a DerivedDimension); likewise, some dimensions may
        # be in the data space but not in the iteration space (e.g., when a
        # function is indexed with integers only)
        return set(self.dspace.dimensions) | set(self.ispace.dimensions)

    @property
    def implicit_dims(self):
        return self._implicit_dims

    @cached_property
    def conditionals(self):
        return self._conditionals or frozendict()

    @property
    def directions(self):
        return self.ispace.directions

    @property
    def dtype(self):
        return self.lhs.dtype

    @cached_property
    def grid(self):
        grids = set()
        for f in self.dspace.parts:
            if f.is_DiscreteFunction:
                grids.add(f.grid)

        if len(grids) == 1:
            return grids.pop()
        else:
            return None

    @property
    def state(self):
        return {i: getattr(self, i) for i in self._state}

class LoweredEq(sympy.Eq, IREq):

    """
    LoweredEq(devito.Eq)
    LoweredEq(devito.LoweredEq, **kwargs)
    LoweredEq(lhs, rhs, **kwargs)

    A SymPy equation with associated IterationSpace and DataSpace.

    When created as ``LoweredEq(devito.Eq)``, the iteration and data spaces are
    automatically derived from analysis of ``expr``.

    When created as ``LoweredEq(devito.LoweredEq, **kwargs)``, the keyword
    arguments can be anything that appears in ``LoweredEq._state`` (i.e.,
    ispace, dspace, ...).

    When created as ``LoweredEq(lhs, rhs, **kwargs)``, *all* keywords in
    ``LoweredEq._state`` must appear in ``kwargs``.
    """

    _state = IREq._state + ('reads', 'writes')

    def __new__(cls, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], LoweredEq):
            # origin: LoweredEq(devito.LoweredEq, **kwargs)
            input_expr = args[0]
            expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.get(i) or getattr(input_expr, i))
            return expr
        elif len(args) == 1 and isinstance(args[0], Eq):
            # origin: LoweredEq(devito.Eq)
            input_expr = expr = args[0]
        elif len(args) == 2:
            expr = sympy.Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
            return expr
        else:
            raise ValueError("Cannot construct LoweredEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))

        # Well-defined dimension ordering
        ordering = dimension_sort(expr)

        # Analyze the expression
        mapper = detect_accesses(expr)
        oobs = detect_oobs(mapper)
        conditional_dimensions = [i for i in ordering if i.is_Conditional]

        # Construct Intervals for IterationSpace and DataSpace
        intervals = build_intervals(Stencil.union(*mapper.values()))
        iintervals = []  # iteration Intervals
        dintervals = []  # data Intervals
        for i in intervals:
            d = i.dim
            if d in oobs:
                iintervals.append(i.zero())
                dintervals.append(i)
            else:
                iintervals.append(i.zero())
                dintervals.append(i.zero())

        # Construct the IterationSpace
        iintervals = IntervalGroup(iintervals, relations=ordering.relations)
        iterators = build_iterators(mapper)
        ispace = IterationSpace(iintervals, iterators)

        # Construct the DataSpace
        dintervals.extend([Interval(i, 0, 0) for i in ordering
                           if i not in ispace.dimensions + conditional_dimensions])
        parts = {k: IntervalGroup(build_intervals(v)).add(iintervals)
                 for k, v in mapper.items() if k}
        dspace = DataSpace(dintervals, parts)

        # Construct the conditionals
        conditionals = {}
        for d in conditional_dimensions:
            if d.condition is None:
                conditionals[d] = CondEq(d.parent % d.factor, 0)
            else:
                conditionals[d] = lower_exprs(d.condition)
        conditionals = frozendict(conditionals)

        # Lower all Differentiable operations into SymPy operations
        rhs = diff2sympy(expr.rhs)

        # Finally create the LoweredEq with all metadata attached
        expr = super(LoweredEq, cls).__new__(cls, expr.lhs, rhs, evaluate=False)

        expr._dspace = dspace
        expr._ispace = ispace
        expr._conditionals = conditionals
        expr._reads, expr._writes = detect_io(expr)

        expr._is_Increment = input_expr.is_Increment
        expr._implicit_dims = input_expr.implicit_dims

        return expr

    @property
    def reads(self):
        return self._reads

    @property
    def writes(self):
        return self._writes

    def xreplace(self, rules):
        return LoweredEq(self.lhs.xreplace(rules), self.rhs.xreplace(rules), **self.state)

    def func(self, *args):
        return super(LoweredEq, self).func(*args, **self.state, evaluate=False)

class ClusterizedEq(sympy.Eq, IREq, Pickable):

    """
    ClusterizedEq(devito.IREq, **kwargs)
    ClusterizedEq(lhs, rhs, **kwargs)

    A SymPy equation with associated IterationSpace and DataSpace.

    There are two main differences between a LoweredEq and a
    ClusterizedEq:

    * In a ClusterizedEq, the iteration and data spaces must *always*
      be provided by the caller.
    * A ClusterizedEq is "frozen", meaning that any call to ``xreplace``
      will not trigger re-evaluation (e.g., mathematical simplification)
      of the expression.

    These two properties make a ClusterizedEq suitable for use in a Cluster.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            # origin: ClusterizedEq(expr, **kwargs)
            input_expr = args[0]
            expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                v = kwargs[i] if i in kwargs else getattr(input_expr, i, None)
                setattr(expr, '_%s' % i, v)
        elif len(args) == 2:
            # origin: ClusterizedEq(lhs, rhs, **kwargs)
            expr = sympy.Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
        else:
            raise ValueError("Cannot construct ClusterizedEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))
        return expr

    def func(self, *args, **kwargs):
        kwargs = {k: kwargs.get(k, v) for k, v in self.state.items()}
        return super(ClusterizedEq, self).func(*args, **kwargs)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = IREq._state
    __reduce_ex__ = Pickable.__reduce_ex__

class DummyEq(ClusterizedEq):

    """
    DummyEq(expr)
    DummyEq(lhs, rhs)

    A special ClusterizedEq with void iteration and data spaces.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            input_expr = args[0]
            assert isinstance(input_expr, Eq)
            obj = LoweredEq(input_expr)
        elif len(args) == 2:
            obj = LoweredEq(Eq(*args, evaluate=False))
        else:
            raise ValueError("Cannot construct DummyEq from args=%s" % str(args))
        return ClusterizedEq.__new__(cls, obj, ispace=obj.ispace, dspace=obj.dspace)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = []
<commit_msg>Conditonals: Added diff2sympy to conditionals<commit_after>from cached_property import cached_property
import sympy

from devito.ir.equations.algorithms import dimension_sort, lower_exprs
from devito.finite_differences.differentiable import diff2sympy
from devito.ir.support import (IterationSpace, DataSpace, Interval, IntervalGroup,
                               Stencil, detect_accesses, detect_oobs, detect_io,
                               build_intervals, build_iterators)
from devito.symbolics import CondEq
from devito.tools import Pickable, frozendict
from devito.types import Eq

__all__ = ['LoweredEq', 'ClusterizedEq', 'DummyEq']

class IREq(object):

    """
    A mixin providing operations common to all :mod:`ir` equation types.
    """
    _state = ('is_Increment', 'ispace', 'dspace', 'conditionals', 'implicit_dims')

    @property
    def is_Scalar(self):
        return self.lhs.is_Symbol

    is_scalar = is_Scalar

    @property
    def is_Tensor(self):
        return self.lhs.is_Indexed

    @property
    def is_Increment(self):
        return self._is_Increment

    @property
    def ispace(self):
        return self._ispace

    @property
    def dspace(self):
        return self._dspace

    @cached_property
    def dimensions(self):
        # Note: some dimensions may be in the iteration space but not in the
        # data space (e.g., a DerivedDimension); likewise, some dimensions may
        # be in the data space but not in the iteration space (e.g., when a
        # function is indexed with integers only)
        return set(self.dspace.dimensions) | set(self.ispace.dimensions)

    @property
    def implicit_dims(self):
        return self._implicit_dims

    @cached_property
    def conditionals(self):
        return self._conditionals or frozendict()

    @property
    def directions(self):
        return self.ispace.directions

    @property
    def dtype(self):
        return self.lhs.dtype

    @cached_property
    def grid(self):
        grids = set()
        for f in self.dspace.parts:
            if f.is_DiscreteFunction:
                grids.add(f.grid)

        if len(grids) == 1:
            return grids.pop()
        else:
            return None

    @property
    def state(self):
        return {i: getattr(self, i) for i in self._state}

class LoweredEq(sympy.Eq, IREq):

    """
    LoweredEq(devito.Eq)
    LoweredEq(devito.LoweredEq, **kwargs)
    LoweredEq(lhs, rhs, **kwargs)

    A SymPy equation with associated IterationSpace and DataSpace.

    When created as ``LoweredEq(devito.Eq)``, the iteration and data spaces are
    automatically derived from analysis of ``expr``.

    When created as ``LoweredEq(devito.LoweredEq, **kwargs)``, the keyword
    arguments can be anything that appears in ``LoweredEq._state`` (i.e.,
    ispace, dspace, ...).

    When created as ``LoweredEq(lhs, rhs, **kwargs)``, *all* keywords in
    ``LoweredEq._state`` must appear in ``kwargs``.
    """

    _state = IREq._state + ('reads', 'writes')

    def __new__(cls, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], LoweredEq):
            # origin: LoweredEq(devito.LoweredEq, **kwargs)
            input_expr = args[0]
            expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.get(i) or getattr(input_expr, i))
            return expr
        elif len(args) == 1 and isinstance(args[0], Eq):
            # origin: LoweredEq(devito.Eq)
            input_expr = expr = args[0]
        elif len(args) == 2:
            expr = sympy.Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
            return expr
        else:
            raise ValueError("Cannot construct LoweredEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))

        # Well-defined dimension ordering
        ordering = dimension_sort(expr)

        # Analyze the expression
        mapper = detect_accesses(expr)
        oobs = detect_oobs(mapper)
        conditional_dimensions = [i for i in ordering if i.is_Conditional]

        # Construct Intervals for IterationSpace and DataSpace
        intervals = build_intervals(Stencil.union(*mapper.values()))
        iintervals = []  # iteration Intervals
        dintervals = []  # data Intervals
        for i in intervals:
            d = i.dim
            if d in oobs:
                iintervals.append(i.zero())
                dintervals.append(i)
            else:
                iintervals.append(i.zero())
                dintervals.append(i.zero())

        # Construct the IterationSpace
        iintervals = IntervalGroup(iintervals, relations=ordering.relations)
        iterators = build_iterators(mapper)
        ispace = IterationSpace(iintervals, iterators)

        # Construct the DataSpace
        dintervals.extend([Interval(i, 0, 0) for i in ordering
                           if i not in ispace.dimensions + conditional_dimensions])
        parts = {k: IntervalGroup(build_intervals(v)).add(iintervals)
                 for k, v in mapper.items() if k}
        dspace = DataSpace(dintervals, parts)

        # Construct the conditionals
        conditionals = {}
        for d in conditional_dimensions:
            if d.condition is None:
                conditionals[d] = CondEq(d.parent % d.factor, 0)
            else:
                conditionals[d] = diff2sympy(lower_exprs(d.condition))
        conditionals = frozendict(conditionals)

        # Lower all Differentiable operations into SymPy operations
        rhs = diff2sympy(expr.rhs)

        # Finally create the LoweredEq with all metadata attached
        expr = super(LoweredEq, cls).__new__(cls, expr.lhs, rhs, evaluate=False)

        expr._dspace = dspace
        expr._ispace = ispace
        expr._conditionals = conditionals
        expr._reads, expr._writes = detect_io(expr)

        expr._is_Increment = input_expr.is_Increment
        expr._implicit_dims = input_expr.implicit_dims

        return expr

    @property
    def reads(self):
        return self._reads

    @property
    def writes(self):
        return self._writes

    def xreplace(self, rules):
        return LoweredEq(self.lhs.xreplace(rules), self.rhs.xreplace(rules), **self.state)

    def func(self, *args):
        return super(LoweredEq, self).func(*args, **self.state, evaluate=False)

class ClusterizedEq(sympy.Eq, IREq, Pickable):

    """
    ClusterizedEq(devito.IREq, **kwargs)
    ClusterizedEq(lhs, rhs, **kwargs)

    A SymPy equation with associated IterationSpace and DataSpace.

    There are two main differences between a LoweredEq and a
    ClusterizedEq:

    * In a ClusterizedEq, the iteration and data spaces must *always*
      be provided by the caller.
    * A ClusterizedEq is "frozen", meaning that any call to ``xreplace``
      will not trigger re-evaluation (e.g., mathematical simplification)
      of the expression.

    These two properties make a ClusterizedEq suitable for use in a Cluster.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            # origin: ClusterizedEq(expr, **kwargs)
            input_expr = args[0]
            expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                v = kwargs[i] if i in kwargs else getattr(input_expr, i, None)
                setattr(expr, '_%s' % i, v)
        elif len(args) == 2:
            # origin: ClusterizedEq(lhs, rhs, **kwargs)
            expr = sympy.Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
        else:
            raise ValueError("Cannot construct ClusterizedEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))
        return expr

    def func(self, *args, **kwargs):
        kwargs = {k: kwargs.get(k, v) for k, v in self.state.items()}
        return super(ClusterizedEq, self).func(*args, **kwargs)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = IREq._state
    __reduce_ex__ = Pickable.__reduce_ex__

class DummyEq(ClusterizedEq):

    """
    DummyEq(expr)
    DummyEq(lhs, rhs)

    A special ClusterizedEq with void iteration and data spaces.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            input_expr = args[0]
            assert isinstance(input_expr, Eq)
            obj = LoweredEq(input_expr)
        elif len(args) == 2:
            obj = LoweredEq(Eq(*args, evaluate=False))
        else:
            raise ValueError("Cannot construct DummyEq from args=%s" % str(args))
        return ClusterizedEq.__new__(cls, obj, ispace=obj.ispace, dspace=obj.dspace)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = []
<|endoftext|>


<commit_before>load("@bazel_skylib//lib:shell.bzl", "shell")

def _buildozer_impl(ctx):
    # That way we don't depend on defaults encoded in the binary but always
    # use defaults set on attributes of the rule
    args = [
        "-k=%s" % str(ctx.attr.keep_going).lower(),
        "-types=%s" % ",".join(ctx.attr.types),
        "-eol-comments=%s" % str(ctx.attr.prefer_eol_comments).lower(),
        "-quiet=%s" % str(ctx.attr.quiet).lower(),
        "-edit-variables=%s" % str(ctx.attr.edit_variables).lower(),
        "-shorten_labels=%s" % str(ctx.attr.shorten_labels).lower(),
        "-delete_with_comments=%s" % str(ctx.attr.delete_with_comments).lower(),
    ]

    if ctx.file.commands:
        args.append("-f=%s" % ctx.file.commands.short_path)
    if ctx.file.tables:
        args.append("-tables=%s" % ctx.file.tables.short_path)
    if ctx.file.add_tables:
        args.append("-add_tables=%s" % ctx.file.add_tables.short_path)

    if ctx.executable.buildifier:
        buildifier_short_path = ctx.executable.buildifier.short_path
    else:
        buildifier_short_path = ""

    out_file = ctx.actions.declare_file(ctx.label.name + ".bash")
    substitutions = {
        "@@ARGS@@": shell.array_literal(args),
        "@@BUILDIFIER_SHORT_PATH@@": shell.quote(buildifier_short_path),
        "@@BUILDOZER_SHORT_PATH@@": shell.quote(ctx.executable._buildozer.short_path),
        "@@ERROR_ON_NO_CHANGES@@": shell.quote(str(ctx.attr.error_on_no_changes).lower()),
        "@@FORMAT_ON_WRITE@@": shell.quote(str(ctx.attr.format_on_write).lower()),
    }
    ctx.actions.expand_template(
        template = ctx.file._runner,
        output = out_file,
        substitutions = substitutions,
        is_executable = True,
    )
    runfiles = ctx.runfiles(files = [ctx.executable._buildozer])
    return [DefaultInfo(
        files = depset([out_file]),
        runfiles = runfiles,
        executable = out_file,
    )]

_buildozer = rule(
    implementation = _buildozer_impl,
    attrs = {
        "commands": attr.label(
            allow_single_file = True,
            doc = "File to read commands from",
        ),
        "keep_going": attr.bool(
            doc = "Apply all commands, even if there are failures",
        ),
        "types": attr.string_list(
            doc = "List of rule types to change, the default empty list means all rules",
        ),
        "prefer_eol_comments": attr.bool(
            doc = "When adding a new comment, put it on the same line if possible",
            default = True,
        ),
        "quiet": attr.bool(
            doc = "Suppress informational messages",
        ),
        "edit_variables": attr.bool(
            doc = "For attributes that simply assign a variable (e.g. hdrs = LIB_HDRS), edit the build variable instead of appending to the attribute",
        ),
        "tables": attr.label(
            doc = "JSON file with custom table definitions which will replace the built-in tables",
            allow_single_file = True,
        ),
        "add_tables": attr.label(
            doc = "JSON file with custom table definitions which will be merged with the built-in tables",
            allow_single_file = True,
        ),
        "shorten_labels": attr.bool(
            doc = "Convert added labels to short form, e.g. //foo:bar => :bar",
            default = True,
        ),
        "delete_with_comments": attr.bool(
            doc = "If a list attribute should be deleted even if there is a comment attached to it",
            default = True,
        ),
        "error_on_no_changes": attr.bool(
            doc = "Exit with 3 on success, when no changes were made",
        ),
        "format_on_write": attr.bool(
            default = False,
            doc = "Set to True for format on write using the -buildifier flag. If the `buildifier` attribute is empty, use the built-in formatter",
        ),
        "buildifier": attr.label(
            cfg = "host",
            executable = True,
            doc = "A label pointing to an executable buildifier output. Has no meaning unless `format_on_write` is True",
        ),
        "_buildozer": attr.label(
            default = "@com_github_bazelbuild_buildtools//buildozer",
            cfg = "host",
            executable = True,
        ),
        "_runner": attr.label(
            default = "@com_github_atlassian_bazel_tools//buildozer:runner.bash.template",
            allow_single_file = True,
        ),
    },
    executable = True,
)

def buildozer(**kwargs):
    tags = kwargs.get("tags", [])
    if "manual" not in tags:
        tags.append("manual")
        kwargs["tags"] = tags
    _buildozer(
        **kwargs
    )
<commit_msg>fix(buildozer): add the buildifier executable to runfiles<commit_after>load("@bazel_skylib//lib:shell.bzl", "shell")

def _buildozer_impl(ctx):
    # That way we don't depend on defaults encoded in the binary but always
    # use defaults set on attributes of the rule
    args = [
        "-k=%s" % str(ctx.attr.keep_going).lower(),
        "-types=%s" % ",".join(ctx.attr.types),
        "-eol-comments=%s" % str(ctx.attr.prefer_eol_comments).lower(),
        "-quiet=%s" % str(ctx.attr.quiet).lower(),
        "-edit-variables=%s" % str(ctx.attr.edit_variables).lower(),
        "-shorten_labels=%s" % str(ctx.attr.shorten_labels).lower(),
        "-delete_with_comments=%s" % str(ctx.attr.delete_with_comments).lower(),
    ]

    runfiles = [ctx.executable._buildozer]

    if ctx.file.commands:
        args.append("-f=%s" % ctx.file.commands.short_path)
    if ctx.file.tables:
        args.append("-tables=%s" % ctx.file.tables.short_path)
    if ctx.file.add_tables:
        args.append("-add_tables=%s" % ctx.file.add_tables.short_path)

    if ctx.executable.buildifier:
        buildifier_short_path = ctx.executable.buildifier.short_path
        runfiles.append(ctx.executable.buildifier)
    else:
        buildifier_short_path = ""

    out_file = ctx.actions.declare_file(ctx.label.name + ".bash")
    substitutions = {
        "@@ARGS@@": shell.array_literal(args),
        "@@BUILDIFIER_SHORT_PATH@@": shell.quote(buildifier_short_path),
        "@@BUILDOZER_SHORT_PATH@@": shell.quote(ctx.executable._buildozer.short_path),
        "@@ERROR_ON_NO_CHANGES@@": shell.quote(str(ctx.attr.error_on_no_changes).lower()),
        "@@FORMAT_ON_WRITE@@": shell.quote(str(ctx.attr.format_on_write).lower()),
    }
    ctx.actions.expand_template(
        template = ctx.file._runner,
        output = out_file,
        substitutions = substitutions,
        is_executable = True,
    )

    return [DefaultInfo(
        files = depset([out_file]),
        runfiles = ctx.runfiles(files = runfiles),
        executable = out_file,
    )]

_buildozer = rule(
    implementation = _buildozer_impl,
    attrs = {
        "commands": attr.label(
            allow_single_file = True,
            doc = "File to read commands from",
        ),
        "keep_going": attr.bool(
            doc = "Apply all commands, even if there are failures",
        ),
        "types": attr.string_list(
            doc = "List of rule types to change, the default empty list means all rules",
        ),
        "prefer_eol_comments": attr.bool(
            doc = "When adding a new comment, put it on the same line if possible",
            default = True,
        ),
        "quiet": attr.bool(
            doc = "Suppress informational messages",
        ),
        "edit_variables": attr.bool(
            doc = "For attributes that simply assign a variable (e.g. hdrs = LIB_HDRS), edit the build variable instead of appending to the attribute",
        ),
        "tables": attr.label(
            doc = "JSON file with custom table definitions which will replace the built-in tables",
            allow_single_file = True,
        ),
        "add_tables": attr.label(
            doc = "JSON file with custom table definitions which will be merged with the built-in tables",
            allow_single_file = True,
        ),
        "shorten_labels": attr.bool(
            doc = "Convert added labels to short form, e.g. //foo:bar => :bar",
            default = True,
        ),
        "delete_with_comments": attr.bool(
            doc = "If a list attribute should be deleted even if there is a comment attached to it",
            default = True,
        ),
        "error_on_no_changes": attr.bool(
            doc = "Exit with 3 on success, when no changes were made",
        ),
        "format_on_write": attr.bool(
            default = False,
            doc = "Set to True for format on write using the -buildifier flag. If the `buildifier` attribute is empty, use the built-in formatter",
        ),
        "buildifier": attr.label(
            cfg = "host",
            executable = True,
            doc = "A label pointing to an executable buildifier output. Has no meaning unless `format_on_write` is True",
        ),
        "_buildozer": attr.label(
            default = "@com_github_bazelbuild_buildtools//buildozer",
            cfg = "host",
            executable = True,
        ),
        "_runner": attr.label(
            default = "@com_github_atlassian_bazel_tools//buildozer:runner.bash.template",
            allow_single_file = True,
        ),
    },
    executable = True,
)

def buildozer(**kwargs):
    tags = kwargs.get("tags", [])
    if "manual" not in tags:
        tags.append("manual")
        kwargs["tags"] = tags
    _buildozer(
        **kwargs
    )
<|endoftext|>


<commit_before>import sys
import gc
import uos as os
import uerrno as errno
import ujson as json
import uzlib
import upip_utarfile as tarfile
gc.collect()

debug = False
install_path = None
cleanup_files = []
gzdict_sz = 16 + 15

file_buf = bytearray(512)

class NotFoundError(Exception):
    pass

def op_split(path):
    if path == "":
        return ("", "")
    r = path.rsplit("/", 1)
    if len(r) == 1:
        return ("", path)
    head = r[0]
    if not head:
        head = "/"
    return (head, r[1])

def op_basename(path):
    return op_split(path)[1]

# Expects *file* name
def _makedirs(name, mode=0o777):
    ret = False
    s = ""
    comps = name.rstrip("/").split("/")[:-1]
    if comps[0] == "":
        s = "/"
    for c in comps:
        if s and s[-1] != "/":
            s += "/"
        s += c
        try:
            os.mkdir(s)
            ret = True
        except OSError as e:
            if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
                raise
            ret = False
    return ret

def save_file(fname, subf):
    global file_buf
    with open(fname, "wb") as outf:
        while True:
            sz = subf.readinto(file_buf)
            if not sz:
                break
            outf.write(file_buf, sz)

def install_tar(f, prefix):
    meta = {}
    for info in f:
        #print(info)
        fname = info.name
        try:
            fname = fname[fname.index("/") + 1:]
        except ValueError:
            fname = ""

        save = True
        for p in ("setup.", "PKG-INFO", "README"):
                #print(fname, p)
                if fname.startswith(p) or ".egg-info" in fname:
                    if fname.endswith("/requires.txt"):
                        meta["deps"] = f.extractfile(info).read()
                    save = False
                    if debug:
                        print("Skipping", fname)
                    break

        if save:
            outfname = prefix + fname
            if info.type != tarfile.DIRTYPE:
                if debug:
                    print("Extracting " + outfname)
                _makedirs(outfname)
                subf = f.extractfile(info)
                save_file(outfname, subf)
    return meta

def expandhome(s):
    if "~/" in s:
        h = os.getenv("HOME")
        s = s.replace("~/", h + "/")
    return s

import ussl
import usocket
warn_ussl = True
def url_open(url):
    global warn_ussl

    if debug:
        print(url)

    proto, _, host, urlpath = url.split('/', 3)
    try:
        ai = usocket.getaddrinfo(host, 443)
    except OSError as e:
        fatal("Unable to resolve %s (no Internet?)" % host, e)
    #print("Address infos:", ai)
    addr = ai[0][4]

    s = usocket.socket(ai[0][0])
    try:
        #print("Connect address:", addr)
        s.connect(addr)

        if proto == "https:":
            s = ussl.wrap_socket(s, server_hostname=host)
            if warn_ussl:
                print("Warning: %s SSL certificate is not validated" % host)
                warn_ussl = False

        # MicroPython rawsocket module supports file interface directly
        s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
        l = s.readline()
        protover, status, msg = l.split(None, 2)
        if status != b"200":
            if status == b"404" or status == b"301":
                raise NotFoundError("Package not found")
            raise ValueError(status)
        while 1:
            l = s.readline()
            if not l:
                raise ValueError("Unexpected EOF in HTTP headers")
            if l == b'\r\n':
                break
    except Exception as e:
        s.close()
        raise e

    return s

def get_pkg_metadata(name):
    f = url_open("https://badge.sha2017.org/eggs/%s/json" % name)
    try:
        return json.load(f)
    finally:
        f.close()

def fatal(msg, exc=None):
    print("Error:", msg)
    if exc and debug:
        raise exc
    sys.exit(1)

def install_pkg(pkg_spec, install_path):
    data = get_pkg_metadata(pkg_spec)
    print("Not (yet) checking if %s already installed" % (pkg_spec))
    latest_ver = data["info"]["version"]
    print("Not (yet) checking if version %s already installed" % (latest_ver))
    packages = data["releases"][latest_ver]
    del data
    gc.collect()
    assert len(packages) == 1
    package_url = packages[0]["url"]
    print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
    package_fname = op_basename(package_url)
    f1 = url_open(package_url)
    try:
        f2 = uzlib.DecompIO(f1, gzdict_sz)
        f3 = tarfile.TarFile(fileobj=f2)
        meta = install_tar(f3, install_path)
    finally:
        f1.close()
    del f3
    del f2
    gc.collect()
    return meta

def install(to_install, install_path=None):
    # Calculate gzip dictionary size to use
    global gzdict_sz
    sz = gc.mem_free() + gc.mem_alloc()
    if sz <= 65536:
        gzdict_sz = 16 + 12

    if install_path is None:
        install_path = get_install_path()
    if install_path[-1] != "/":
        install_path += "/"
    if not isinstance(to_install, list):
        to_install = [to_install]
    print("Installing to: " + install_path)
    # sets would be perfect here, but don't depend on them
    installed = []
    try:
        while to_install:
            if debug:
                print("Queue:", to_install)
            pkg_spec = to_install.pop(0)
            if pkg_spec in installed:
                continue
            meta = install_pkg(pkg_spec, install_path)
            installed.append(pkg_spec)
            if debug:
                print(meta)
            deps = meta.get("deps", "").rstrip()
            if deps:
                deps = deps.decode("utf-8").split("\n")
                to_install.extend(deps)
    except Exception as e:
        print("Error installing '{}': {}, packages may be partially installed".format(
                pkg_spec, e),
            file=sys.stderr)

def get_install_path():
    global install_path
    if install_path is None:
        # sys.path[0] is current module's path
        install_path = sys.path[1]
    install_path = expandhome(install_path)
    return install_path

def cleanup():
    for fname in cleanup_files:
        try:
            os.unlink(fname)
        except OSError:
            print("Warning: Cannot delete " + fname)

def help():
    print("""\
woezel - Clone of the Simple PyPI package manager for MicroPython
Usage: micropython -m woezel install [-p <path>] <package>... | -r <requirements.txt>
import woezel; shapip.install(package_or_list, [<path>])

If <path> is not given, packages will be installed into sys.path[1]
(can be set from MICROPYPATH environment variable, if current system
supports that).""")
    print("Current value of sys.path[1]:", sys.path[1])
    print("""\

Note: only MicroPython packages are supported for installation,
woezel, like upip does not support arbitrary code in setup.py.
""")

def main():
    global debug
    global install_path
    install_path = None

    if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
        help()
        return

    if sys.argv[1] != "install":
        fatal("Only 'install' command supported")

    to_install = []

    i = 2
    while i < len(sys.argv) and sys.argv[i][0] == "-":
        opt = sys.argv[i]
        i += 1
        if opt == "-h" or opt == "--help":
            help()
            return
        elif opt == "-p":
            install_path = sys.argv[i]
            i += 1
        elif opt == "-r":
            list_file = sys.argv[i]
            i += 1
            with open(list_file) as f:
                while True:
                    l = f.readline()
                    if not l:
                        break
                    if l[0] == "#":
                        continue
                    to_install.append(l.rstrip())
        elif opt == "--debug":
            debug = True
        else:
            fatal("Unknown/unsupported option: " + opt)

    to_install.extend(sys.argv[i:])
    if not to_install:
        help()
        return

    install(to_install)

    if not debug:
        cleanup()

if __name__ == "__main__":
    main()
<commit_msg>Woezel /eggs/get/[name]/json<commit_after>import sys
import gc
import uos as os
import uerrno as errno
import ujson as json
import uzlib
import upip_utarfile as tarfile
gc.collect()

debug = False
install_path = None
cleanup_files = []
gzdict_sz = 16 + 15

file_buf = bytearray(512)

class NotFoundError(Exception):
    pass

def op_split(path):
    if path == "":
        return ("", "")
    r = path.rsplit("/", 1)
    if len(r) == 1:
        return ("", path)
    head = r[0]
    if not head:
        head = "/"
    return (head, r[1])

def op_basename(path):
    return op_split(path)[1]

# Expects *file* name
def _makedirs(name, mode=0o777):
    ret = False
    s = ""
    comps = name.rstrip("/").split("/")[:-1]
    if comps[0] == "":
        s = "/"
    for c in comps:
        if s and s[-1] != "/":
            s += "/"
        s += c
        try:
            os.mkdir(s)
            ret = True
        except OSError as e:
            if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
                raise
            ret = False
    return ret

def save_file(fname, subf):
    global file_buf
    with open(fname, "wb") as outf:
        while True:
            sz = subf.readinto(file_buf)
            if not sz:
                break
            outf.write(file_buf, sz)

def install_tar(f, prefix):
    meta = {}
    for info in f:
        #print(info)
        fname = info.name
        try:
            fname = fname[fname.index("/") + 1:]
        except ValueError:
            fname = ""

        save = True
        for p in ("setup.", "PKG-INFO", "README"):
                #print(fname, p)
                if fname.startswith(p) or ".egg-info" in fname:
                    if fname.endswith("/requires.txt"):
                        meta["deps"] = f.extractfile(info).read()
                    save = False
                    if debug:
                        print("Skipping", fname)
                    break

        if save:
            outfname = prefix + fname
            if info.type != tarfile.DIRTYPE:
                if debug:
                    print("Extracting " + outfname)
                _makedirs(outfname)
                subf = f.extractfile(info)
                save_file(outfname, subf)
    return meta

def expandhome(s):
    if "~/" in s:
        h = os.getenv("HOME")
        s = s.replace("~/", h + "/")
    return s

import ussl
import usocket
warn_ussl = True
def url_open(url):
    global warn_ussl

    if debug:
        print(url)

    proto, _, host, urlpath = url.split('/', 3)
    try:
        ai = usocket.getaddrinfo(host, 443)
    except OSError as e:
        fatal("Unable to resolve %s (no Internet?)" % host, e)
    #print("Address infos:", ai)
    addr = ai[0][4]

    s = usocket.socket(ai[0][0])
    try:
        #print("Connect address:", addr)
        s.connect(addr)

        if proto == "https:":
            s = ussl.wrap_socket(s, server_hostname=host)
            if warn_ussl:
                print("Warning: %s SSL certificate is not validated" % host)
                warn_ussl = False

        # MicroPython rawsocket module supports file interface directly
        s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
        l = s.readline()
        protover, status, msg = l.split(None, 2)
        if status != b"200":
            if status == b"404" or status == b"301":
                raise NotFoundError("Package not found")
            raise ValueError(status)
        while 1:
            l = s.readline()
            if not l:
                raise ValueError("Unexpected EOF in HTTP headers")
            if l == b'\r\n':
                break
    except Exception as e:
        s.close()
        raise e

    return s

def get_pkg_metadata(name):
    f = url_open("https://badge.sha2017.org/eggs/get/%s/json" % name)
    try:
        return json.load(f)
    finally:
        f.close()

def fatal(msg, exc=None):
    print("Error:", msg)
    if exc and debug:
        raise exc
    sys.exit(1)

def install_pkg(pkg_spec, install_path):
    data = get_pkg_metadata(pkg_spec)
    print("Not (yet) checking if %s already installed" % (pkg_spec))
    latest_ver = data["info"]["version"]
    print("Not (yet) checking if version %s already installed" % (latest_ver))
    packages = data["releases"][latest_ver]
    del data
    gc.collect()
    assert len(packages) == 1
    package_url = packages[0]["url"]
    print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
    package_fname = op_basename(package_url)
    f1 = url_open(package_url)
    try:
        f2 = uzlib.DecompIO(f1, gzdict_sz)
        f3 = tarfile.TarFile(fileobj=f2)
        meta = install_tar(f3, install_path)
    finally:
        f1.close()
    del f3
    del f2
    gc.collect()
    return meta

def install(to_install, install_path=None):
    # Calculate gzip dictionary size to use
    global gzdict_sz
    sz = gc.mem_free() + gc.mem_alloc()
    if sz <= 65536:
        gzdict_sz = 16 + 12

    if install_path is None:
        install_path = get_install_path()
    if install_path[-1] != "/":
        install_path += "/"
    if not isinstance(to_install, list):
        to_install = [to_install]
    print("Installing to: " + install_path)
    # sets would be perfect here, but don't depend on them
    installed = []
    try:
        while to_install:
            if debug:
                print("Queue:", to_install)
            pkg_spec = to_install.pop(0)
            if pkg_spec in installed:
                continue
            meta = install_pkg(pkg_spec, install_path)
            installed.append(pkg_spec)
            if debug:
                print(meta)
            deps = meta.get("deps", "").rstrip()
            if deps:
                deps = deps.decode("utf-8").split("\n")
                to_install.extend(deps)
    except Exception as e:
        print("Error installing '{}': {}, packages may be partially installed".format(
                pkg_spec, e),
            file=sys.stderr)

def get_install_path():
    global install_path
    if install_path is None:
        # sys.path[0] is current module's path
        install_path = sys.path[1]
    install_path = expandhome(install_path)
    return install_path

def cleanup():
    for fname in cleanup_files:
        try:
            os.unlink(fname)
        except OSError:
            print("Warning: Cannot delete " + fname)

def help():
    print("""\
woezel - Clone of the Simple PyPI package manager for MicroPython
Usage: micropython -m woezel install [-p <path>] <package>... | -r <requirements.txt>
import woezel; shapip.install(package_or_list, [<path>])

If <path> is not given, packages will be installed into sys.path[1]
(can be set from MICROPYPATH environment variable, if current system
supports that).""")
    print("Current value of sys.path[1]:", sys.path[1])
    print("""\

Note: only MicroPython packages are supported for installation,
woezel, like upip does not support arbitrary code in setup.py.
""")

def main():
    global debug
    global install_path
    install_path = None

    if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
        help()
        return

    if sys.argv[1] != "install":
        fatal("Only 'install' command supported")

    to_install = []

    i = 2
    while i < len(sys.argv) and sys.argv[i][0] == "-":
        opt = sys.argv[i]
        i += 1
        if opt == "-h" or opt == "--help":
            help()
            return
        elif opt == "-p":
            install_path = sys.argv[i]
            i += 1
        elif opt == "-r":
            list_file = sys.argv[i]
            i += 1
            with open(list_file) as f:
                while True:
                    l = f.readline()
                    if not l:
                        break
                    if l[0] == "#":
                        continue
                    to_install.append(l.rstrip())
        elif opt == "--debug":
            debug = True
        else:
            fatal("Unknown/unsupported option: " + opt)

    to_install.extend(sys.argv[i:])
    if not to_install:
        help()
        return

    install(to_install)

    if not debug:
        cleanup()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import nibabel
import numpy

def ground_thruth_filenames(input_dir):
    nifti_file = input_dir + '_ground_truth.nii.gz'
    reoriented_nifti_file = input_dir + '_ground_truth_reoriented.nii.gz'
    bval_file = input_dir + '_ground_truth.bval'
    bvec_file = input_dir + '_ground_truth.bvec'
    return nifti_file, reoriented_nifti_file, bval_file, bvec_file

def compare_nifti(nifti_file_1, nifti_file_2):
    print(nifti_file_1, nifti_file_2)
    nifti_1 = nibabel.load(nifti_file_1)
    nifti_2 = nibabel.load(nifti_file_2)

    # check the affine
    if not (nifti_1.get_affine() == nifti_2.get_affine()).all():
        print('affine mismatch')
        return False

    # check the data
    if nifti_1.get_data_dtype() != nifti_2.get_data_dtype():
        print('dtype mismatch')
        return False
    if not (nifti_1.get_data() == nifti_2.get_data()).all():
        print('data mismatch')
        return False

    return True

def compare_bval(bval_file_1, bval_file_2):
    bval_1 = numpy.loadtxt(bval_file_1)
    bval_2 = numpy.loadtxt(bval_file_2)
    return (bval_1 == bval_2).all()

def compare_bvec(bvec_file_1, bvec_file_2):
    bvec_1 = numpy.loadtxt(bvec_file_1)
    bvec_2 = numpy.loadtxt(bvec_file_2)
    return (bvec_1 == bvec_2).all()
<commit_msg>MSMET-929: Fix unit tests again after code restyling<commit_after>import nibabel
import numpy

def ground_thruth_filenames(input_dir):
    nifti_file = input_dir + '_ground_truth.nii.gz'
    reoriented_nifti_file = input_dir + '_ground_truth_reoriented.nii.gz'
    bval_file = input_dir + '_ground_truth.bval'
    bvec_file = input_dir + '_ground_truth.bvec'
    return nifti_file, reoriented_nifti_file, bval_file, bvec_file

def compare_nifti(nifti_file_1, nifti_file_2):
    print(nifti_file_1, nifti_file_2)
    nifti_1 = nibabel.load(nifti_file_1)
    nifti_2 = nibabel.load(nifti_file_2)

    # check the affine
    if not (nifti_1.get_affine() == nifti_2.get_affine()).all():
        print('affine mismatch')
        return False

    # check the data
    if nifti_1.get_data_dtype() != nifti_2.get_data_dtype():
        print('dtype mismatch')
        return False
    if not (nifti_1.get_data() == nifti_2.get_data()).all():
        print('data mismatch')
        return False

    return True

def compare_bval(bval_file_1, bval_file_2):
    bval_1 = numpy.loadtxt(bval_file_1)
    bval_2 = numpy.loadtxt(bval_file_2)
    return numpy.array_equal(bval_1, bval_2)

def compare_bvec(bvec_file_1, bvec_file_2):
    bvec_1 = numpy.loadtxt(bvec_file_1)
    bvec_2 = numpy.loadtxt(bvec_file_2)
    return numpy.array_equal(bvec_1, bvec_2)
<|endoftext|>


<commit_before>"""Parse (absolute and relative) URLs.

See RFC 1808: "Relative Uniform Resource Locators", by R. Fielding,
UC Irvine, June 1995.
"""

__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
           "urlsplit", "urlunsplit"]

# A classification of schemes ('' means apply by default)
uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap',
                               'wais', 'file', 'https', 'shttp', 'mms',
                               'prospero', 'rtsp', 'rtspu', '']
uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet',
                             'imap', 'wais', 'file', 'mms', 'https', 'shttp',
                             'snews', 'prospero', 'rtsp', 'rtspu', '']
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
                                  'telnet', 'wais', 'imap', 'snews', 'sip']
uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap',
                             'https', 'shttp', 'rtsp', 'rtspu', 'sip',
                             'mms', '']
uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms',
                            'gopher', 'rtsp', 'rtspu', 'sip', '']
uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news',
                               'nntp', 'wais', 'https', 'shttp', 'snews',
                               'file', 'prospero', '']

# Characters valid in scheme names
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                '0123456789'
                '+-.')

MAX_CACHE_SIZE = 20
_parse_cache = {}

def clear_cache():
    """Clear the parse cache."""
    global _parse_cache
    _parse_cache = {}

def urlparse(url, scheme='', allow_fragments=1):
    """Parse a URL into 6 components:
    <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
    Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    tuple = urlsplit(url, scheme, allow_fragments)
    scheme, netloc, url, query, fragment = tuple
    if scheme in uses_params and ';' in url:
        url, params = _splitparams(url)
    else:
        params = ''
    return scheme, netloc, url, params, query, fragment

def _splitparams(url):
    if '/'  in url:
        i = url.find(';', url.rfind('/'))
        if i < 0:
            return url, ''
    else:
        i = url.find(';')
    return url[:i], url[i+1:]

def urlsplit(url, scheme='', allow_fragments=1):
    """Parse a URL into 5 components:
    <scheme>://<netloc>/<path>?<query>#<fragment>
    Return a 5-tuple: (scheme, netloc, path, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    key = url, scheme, allow_fragments
    cached = _parse_cache.get(key, None)
    if cached:
        return cached
    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
        clear_cache()
    netloc = query = fragment = ''
    i = url.find(':')
    if i > 0:
        if url[:i] == 'http': # optimize the common case
            scheme = url[:i].lower()
            url = url[i+1:]
            if url[:2] == '//':
                i = url.find('/', 2)
                if i < 0:
                    i = url.find('#')
                    if i < 0:
                        i = len(url)
                netloc = url[2:i]
                url = url[i:]
            if allow_fragments and '#' in url:
                url, fragment = url.split('#', 1)
            if '?' in url:
                url, query = url.split('?', 1)
            tuple = scheme, netloc, url, query, fragment
            _parse_cache[key] = tuple
            return tuple
        for c in url[:i]:
            if c not in scheme_chars:
                break
        else:
            scheme, url = url[:i].lower(), url[i+1:]
    if scheme in uses_netloc:
        if url[:2] == '//':
            i = url.find('/', 2)
            if i < 0:
                i = len(url)
            netloc, url = url[2:i], url[i:]
    if allow_fragments and scheme in uses_fragment and '#' in url:
        url, fragment = url.split('#', 1)
    if scheme in uses_query and '?' in url:
        url, query = url.split('?', 1)
    tuple = scheme, netloc, url, query, fragment
    _parse_cache[key] = tuple
    return tuple

def urlunparse((scheme, netloc, url, params, query, fragment)):
    """Put a parsed URL back together again.  This may result in a
    slightly different, but equivalent URL, if the URL that was parsed
    originally had redundant delimiters, e.g. a ? with an empty query
    (the draft states that these are equivalent)."""
    if params:
        url = "%s;%s" % (url, params)
    return urlunsplit((scheme, netloc, url, query, fragment))

def urlunsplit((scheme, netloc, url, query, fragment)):
    if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):
        if url and url[:1] != '/': url = '/' + url
        url = '//' + (netloc or '') + url
    if scheme:
        url = scheme + ':' + url
    if query:
        url = url + '?' + query
    if fragment:
        url = url + '#' + fragment
    return url

def urljoin(base, url, allow_fragments = 1):
    """Join a base URL and a possibly relative URL to form an absolute
    interpretation of the latter."""
    if not base:
        return url
    if not url:
        return base
    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
            urlparse(base, '', allow_fragments)
    scheme, netloc, path, params, query, fragment = \
            urlparse(url, bscheme, allow_fragments)
    if scheme != bscheme or scheme not in uses_relative:
        return url
    if scheme in uses_netloc:
        if netloc:
            return urlunparse((scheme, netloc, path,
                               params, query, fragment))
        netloc = bnetloc
    if path[:1] == '/':
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    if not (path or params or query):
        return urlunparse((scheme, netloc, bpath,
                           bparams, bquery, fragment))
    segments = bpath.split('/')[:-1] + path.split('/')
    # XXX The stuff below is bogus in various ways...
    if segments[-1] == '.':
        segments[-1] = ''
    while '.' in segments:
        segments.remove('.')
    while 1:
        i = 1
        n = len(segments) - 1
        while i < n:
            if (segments[i] == '..'
                and segments[i-1] not in ('', '..')):
                del segments[i-1:i+1]
                break
            i = i+1
        else:
            break
    if segments == ['', '..']:
        segments[-1] = ''
    elif len(segments) >= 2 and segments[-1] == '..':
        segments[-2:] = ['']
    return urlunparse((scheme, netloc, '/'.join(segments),
                       params, query, fragment))

def urldefrag(url):
    """Removes any existing fragment from URL.

    Returns a tuple of the defragmented URL and the fragment.  If
    the URL contained no fragments, the second element is the
    empty string.
    """
    if '#' in url:
        s, n, p, a, q, frag = urlparse(url)
        defrag = urlunparse((s, n, p, a, q, ''))
        return defrag, frag
    else:
        return url, ''

test_input = """
      http://a/b/c/d

      g:h        = <URL:g:h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      g          = <URL:http://a/b/c/g>
      ./g        = <URL:http://a/b/c/g>
      g/         = <URL:http://a/b/c/g/>
      /g         = <URL:http://a/g>
      //g        = <URL:http://g>
      ?y         = <URL:http://a/b/c/d?y>
      g?y        = <URL:http://a/b/c/g?y>
      g?y/./x    = <URL:http://a/b/c/g?y/./x>
      .          = <URL:http://a/b/c/>
      ./         = <URL:http://a/b/c/>
      ..         = <URL:http://a/b/>
      ../        = <URL:http://a/b/>
      ../g       = <URL:http://a/b/g>
      ../..      = <URL:http://a/>
      ../../g    = <URL:http://a/g>
      ../../../g = <URL:http://a/../g>
      ./../g     = <URL:http://a/b/g>
      ./g/.      = <URL:http://a/b/c/g/>
      /./g       = <URL:http://a/./g>
      g/./h      = <URL:http://a/b/c/g/h>
      g/../h     = <URL:http://a/b/c/h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      http:?y         = <URL:http://a/b/c/d?y>
      http:g?y        = <URL:http://a/b/c/g?y>
      http:g?y/./x    = <URL:http://a/b/c/g?y/./x>
"""

def test():
    import sys
    base = ''
    if sys.argv[1:]:
        fn = sys.argv[1]
        if fn == '-':
            fp = sys.stdin
        else:
            fp = open(fn)
    else:
        import StringIO
        fp = StringIO.StringIO(test_input)
    while 1:
        line = fp.readline()
        if not line: break
        words = line.split()
        if not words:
            continue
        url = words[0]
        parts = urlparse(url)
        print '%-10s : %s' % (url, parts)
        abs = urljoin(base, url)
        if not base:
            base = abs
        wrapped = '<URL:%s>' % abs
        print '%-10s = %s' % (url, wrapped)
        if len(words) == 3 and words[1] == '=':
            if wrapped != words[2]:
                print 'EXPECTED', words[2], '!!!!!!!!!!'

if __name__ == '__main__':
    test()
<commit_msg>Use sets instead of lists for membership testing<commit_after>"""Parse (absolute and relative) URLs.

See RFC 1808: "Relative Uniform Resource Locators", by R. Fielding,
UC Irvine, June 1995.
"""

__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
           "urlsplit", "urlunsplit"]

# A classification of schemes ('' means apply by default)
uses_relative = set(['ftp', 'http', 'gopher', 'nntp', 'imap',
                               'wais', 'file', 'https', 'shttp', 'mms',
                               'prospero', 'rtsp', 'rtspu', ''])
uses_netloc = set(['ftp', 'http', 'gopher', 'nntp', 'telnet',
                             'imap', 'wais', 'file', 'mms', 'https', 'shttp',
                             'snews', 'prospero', 'rtsp', 'rtspu', ''])
non_hierarchical = set(['gopher', 'hdl', 'mailto', 'news',
                                  'telnet', 'wais', 'imap', 'snews', 'sip'])
uses_params = set(['ftp', 'hdl', 'prospero', 'http', 'imap',
                             'https', 'shttp', 'rtsp', 'rtspu', 'sip',
                             'mms', ''])
uses_query = set(['http', 'wais', 'imap', 'https', 'shttp', 'mms',
                            'gopher', 'rtsp', 'rtspu', 'sip', ''])
uses_fragment = set(['ftp', 'hdl', 'http', 'gopher', 'news',
                               'nntp', 'wais', 'https', 'shttp', 'snews',
                               'file', 'prospero', ''])

# Characters valid in scheme names
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                '0123456789'
                '+-.')

MAX_CACHE_SIZE = 20
_parse_cache = {}

def clear_cache():
    """Clear the parse cache."""
    global _parse_cache
    _parse_cache = {}

def urlparse(url, scheme='', allow_fragments=1):
    """Parse a URL into 6 components:
    <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
    Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    tuple = urlsplit(url, scheme, allow_fragments)
    scheme, netloc, url, query, fragment = tuple
    if scheme in uses_params and ';' in url:
        url, params = _splitparams(url)
    else:
        params = ''
    return scheme, netloc, url, params, query, fragment

def _splitparams(url):
    if '/'  in url:
        i = url.find(';', url.rfind('/'))
        if i < 0:
            return url, ''
    else:
        i = url.find(';')
    return url[:i], url[i+1:]

def urlsplit(url, scheme='', allow_fragments=1):
    """Parse a URL into 5 components:
    <scheme>://<netloc>/<path>?<query>#<fragment>
    Return a 5-tuple: (scheme, netloc, path, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    key = url, scheme, allow_fragments
    cached = _parse_cache.get(key, None)
    if cached:
        return cached
    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
        clear_cache()
    netloc = query = fragment = ''
    i = url.find(':')
    if i > 0:
        if url[:i] == 'http': # optimize the common case
            scheme = url[:i].lower()
            url = url[i+1:]
            if url[:2] == '//':
                i = url.find('/', 2)
                if i < 0:
                    i = url.find('#')
                    if i < 0:
                        i = len(url)
                netloc = url[2:i]
                url = url[i:]
            if allow_fragments and '#' in url:
                url, fragment = url.split('#', 1)
            if '?' in url:
                url, query = url.split('?', 1)
            tuple = scheme, netloc, url, query, fragment
            _parse_cache[key] = tuple
            return tuple
        for c in url[:i]:
            if c not in scheme_chars:
                break
        else:
            scheme, url = url[:i].lower(), url[i+1:]
    if scheme in uses_netloc:
        if url[:2] == '//':
            i = url.find('/', 2)
            if i < 0:
                i = len(url)
            netloc, url = url[2:i], url[i:]
    if allow_fragments and scheme in uses_fragment and '#' in url:
        url, fragment = url.split('#', 1)
    if scheme in uses_query and '?' in url:
        url, query = url.split('?', 1)
    tuple = scheme, netloc, url, query, fragment
    _parse_cache[key] = tuple
    return tuple

def urlunparse((scheme, netloc, url, params, query, fragment)):
    """Put a parsed URL back together again.  This may result in a
    slightly different, but equivalent URL, if the URL that was parsed
    originally had redundant delimiters, e.g. a ? with an empty query
    (the draft states that these are equivalent)."""
    if params:
        url = "%s;%s" % (url, params)
    return urlunsplit((scheme, netloc, url, query, fragment))

def urlunsplit((scheme, netloc, url, query, fragment)):
    if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):
        if url and url[:1] != '/': url = '/' + url
        url = '//' + (netloc or '') + url
    if scheme:
        url = scheme + ':' + url
    if query:
        url = url + '?' + query
    if fragment:
        url = url + '#' + fragment
    return url

def urljoin(base, url, allow_fragments = 1):
    """Join a base URL and a possibly relative URL to form an absolute
    interpretation of the latter."""
    if not base:
        return url
    if not url:
        return base
    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
            urlparse(base, '', allow_fragments)
    scheme, netloc, path, params, query, fragment = \
            urlparse(url, bscheme, allow_fragments)
    if scheme != bscheme or scheme not in uses_relative:
        return url
    if scheme in uses_netloc:
        if netloc:
            return urlunparse((scheme, netloc, path,
                               params, query, fragment))
        netloc = bnetloc
    if path[:1] == '/':
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    if not (path or params or query):
        return urlunparse((scheme, netloc, bpath,
                           bparams, bquery, fragment))
    segments = bpath.split('/')[:-1] + path.split('/')
    # XXX The stuff below is bogus in various ways...
    if segments[-1] == '.':
        segments[-1] = ''
    while '.' in segments:
        segments.remove('.')
    while 1:
        i = 1
        n = len(segments) - 1
        while i < n:
            if (segments[i] == '..'
                and segments[i-1] not in ('', '..')):
                del segments[i-1:i+1]
                break
            i = i+1
        else:
            break
    if segments == ['', '..']:
        segments[-1] = ''
    elif len(segments) >= 2 and segments[-1] == '..':
        segments[-2:] = ['']
    return urlunparse((scheme, netloc, '/'.join(segments),
                       params, query, fragment))

def urldefrag(url):
    """Removes any existing fragment from URL.

    Returns a tuple of the defragmented URL and the fragment.  If
    the URL contained no fragments, the second element is the
    empty string.
    """
    if '#' in url:
        s, n, p, a, q, frag = urlparse(url)
        defrag = urlunparse((s, n, p, a, q, ''))
        return defrag, frag
    else:
        return url, ''

test_input = """
      http://a/b/c/d

      g:h        = <URL:g:h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      g          = <URL:http://a/b/c/g>
      ./g        = <URL:http://a/b/c/g>
      g/         = <URL:http://a/b/c/g/>
      /g         = <URL:http://a/g>
      //g        = <URL:http://g>
      ?y         = <URL:http://a/b/c/d?y>
      g?y        = <URL:http://a/b/c/g?y>
      g?y/./x    = <URL:http://a/b/c/g?y/./x>
      .          = <URL:http://a/b/c/>
      ./         = <URL:http://a/b/c/>
      ..         = <URL:http://a/b/>
      ../        = <URL:http://a/b/>
      ../g       = <URL:http://a/b/g>
      ../..      = <URL:http://a/>
      ../../g    = <URL:http://a/g>
      ../../../g = <URL:http://a/../g>
      ./../g     = <URL:http://a/b/g>
      ./g/.      = <URL:http://a/b/c/g/>
      /./g       = <URL:http://a/./g>
      g/./h      = <URL:http://a/b/c/g/h>
      g/../h     = <URL:http://a/b/c/h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      http:?y         = <URL:http://a/b/c/d?y>
      http:g?y        = <URL:http://a/b/c/g?y>
      http:g?y/./x    = <URL:http://a/b/c/g?y/./x>
"""

def test():
    import sys
    base = ''
    if sys.argv[1:]:
        fn = sys.argv[1]
        if fn == '-':
            fp = sys.stdin
        else:
            fp = open(fn)
    else:
        import StringIO
        fp = StringIO.StringIO(test_input)
    while 1:
        line = fp.readline()
        if not line: break
        words = line.split()
        if not words:
            continue
        url = words[0]
        parts = urlparse(url)
        print '%-10s : %s' % (url, parts)
        abs = urljoin(base, url)
        if not base:
            base = abs
        wrapped = '<URL:%s>' % abs
        print '%-10s = %s' % (url, wrapped)
        if len(words) == 3 and words[1] == '=':
            if wrapped != words[2]:
                print 'EXPECTED', words[2], '!!!!!!!!!!'

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>__version__ = "0.8.6"
<commit_msg>version 0.8.7<commit_after>__version__ = "0.8.7"
<|endoftext|>


<commit_before>#!/usr/local/bin/python
from azure.common.credentials import ServicePrincipalCredentials
from azure.common.credentials import UserPassCredentials
from azure.common.credentials import BasicTokenAuthentication
from azure.keyvault import KeyVaultClient
from azure.mgmt.keyvault import KeyVaultManagementClient
from adal import token_cache
import adal
import json
import os

CLIENT_ID = '8fd4d3c4-efea-49aa-b1de-2c33c22da56e' # Azure cli
KEY_VAULT_RESOURCE = 'https://vault.azure.net'
AZURE_MANAGEMENT_RESOURCE2 = 'https://management.core.windows.net/'

def _json_format(obj):
    return json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': '))

class KV_Config(dict):
    def __init__(self):

        self.authority_url = ''
        self.subscription_id = ''
        self.tenant_id = ''
        self.token_cache = ''
        self.user_id = ''
        self.resource_group = ''
        self.user_oid = ''

    def __getattr__(self, item):
        return self[item]

    def __setattr__(self, key, value):
        self[key] = value

    def from_disk(self):
        if os.path.isfile('kvconfig.json'):
            with open('kvconfig.json', 'r') as configFile:
                try:
                    dict = json.load(configFile)
                except json.JSONDecodeError:
                    print('error loading config file')
                    return
                for key, value in dict.items():
                    if value:
                        self[key] = value

    def to_disk(self):
        with open('kvconfig.json', 'w') as configFile:
            json.dump(self, configFile, sort_keys=True, indent=4, separators=(',', ': '))

class KV_Auth(object):
    def __init__(self, config):
        self._config = config

        self._cache = token_cache.TokenCache()

        if self._config.token_cache:
            self._cache.deserialize(self._config.token_cache)

        self._context = adal.AuthenticationContext(self._config.authority_url, cache=self._cache)

    def get_keyvault_creds(self):
        return self._get_creds(KEY_VAULT_RESOURCE)

    def get_arm_creds(self):
        return self._get_creds(AZURE_MANAGEMENT_RESOURCE2)

    def _get_auth_token_from_code(self, resource):
        code = self._context.acquire_user_code(resource, CLIENT_ID)

        print(code['message'])

        token = self._context.acquire_token_with_device_code(resource, code, CLIENT_ID)

        return token

    def _get_creds(self, resource):
        token = None

        if not self._config.user_id:
            token = self._get_auth_token_from_code(resource)
        else:
            token = self._context.acquire_token(resource, self._config.user_id, CLIENT_ID)

            if not token:
                token = self._get_auth_token_from_code(resource)

        self._config.token_cache = self._cache.serialize()

        if token:
            if not self._config.user_id:
                self._config.user_id = token['userId']

            if not self._config.user_oid:
                self._config.user_oid = token['oid']

            token['access_token'] = token['accessToken']

            return BasicTokenAuthentication(token)

        return None

class KV_Repl(object):
    _repl_break_commands = set(('quit', 'q', 'back', 'b'))

    def __init__(self, config):
        self._auth = KV_Auth(config)
        self._config = config
        self._mgmt_client = KeyVaultManagementClient(self._auth.get_arm_creds(), config.subscription_id)
        self._data_client = KeyVaultClient(self._auth.get_keyvault_creds())

    def start(self):
        self._vault_index();

    def _vault_index(self):
        selection = None

        exit_loop = False

        while not exit_loop:
            print('\nAvailable Vaults:\n')

            vaults = self._get_vault_list()

            for idx, vault in enumerate(vaults):
                print('%d. %s' % (idx, vault.name))

            print('\n#:select vault (a)dd (d)elete (q)uit\n')

            selection = input('> ')

            if selection == 'a' or selection == 'add':
                continue
            else:
                try:
                    i = int(selection)

                except ValueError:
                    print('invalid input')
                    continue

                selection = self._vault_details(vaults[i])

            if selection == 'q' or selection == 'quit':
                exit_loop = True
                continue

    def _add_vault(self):
        name = input('\nenter vault name:')

    def _vault_details(self, vault):
        selection = None

        exit_loop = False

        while not exit_loop:
            vault_info = self._mgmt_client.vaults.get(self._config.resource_group, vault.name)

            print('\nName:\t%s' % vault_info.name)

            print('Uri:\t%s' % vault_info.properties.vault_uri)

            print('Id:\t%s' % vault_info.id)

            print('\n(s)ecrets (k)eys (c)ertificates (b)ack (q)uit\n')

            selection = input('> ')

            if selection == 's' or selection == 'secrets':
                selection = self._secret_index(vault_info)
            elif selection == 'k' or selection == 'keys':
                print('\nnot yet implemented\n')
                continue
            elif selection == 'c' or selection == 'certificates':
                print('\nnot yet implemented\n')
                continue
            else:
                print('invalid input')
                continue

            if selection == 'b' or selection == 'back':
                selection = None
                exit_loop = True
                continue
            elif selection == 'q' or selection == 'quit':
                exit_loop = True
                continue

        return selection

    def _secret_index(self, vault_info):
        selection = None
        exit_loop = False

        while not exit_loop:
            print('\n%s Secrets:\n' % vault_info.name)

            secrets = [secret for secret in self._data_client.get_secrets(vault_info.properties.vault_uri)]

            for idx, s in enumerate(secrets):
                print('%d. %s' % (idx, KV_Repl._get_secret_name_from_url(s.id)))

            print('\n#:show secret value (a)dd (d)elete (b)ack (q)uit\n')

            selection = input('> ')

            if selection == 'a' or selection == 'add':
                self._add_secret(vault_info)
                continue
            elif selection == 'd' or selection == 'delete':
                print('\nnot yet implemented\n')
                continue
            elif selection == 'b' or selection == 'back' or selection == 'q' or selection == 'quit':
                exit_loop = True
                continue
            else:
                try:
                    i = int(selection)

                except ValueError:
                    i = -1

                if i < 0 and i >= len(secrets):
                    print('invalid input')
                    continue

                print('%s = %s' % (KV_Repl._get_secret_name_from_url(secrets[i].id), self._data_client.get_secret(secrets[i].id).value))

        return selection

    def _add_secret(self, vault_info):
        secret_name = input('\nSecret Name: ')
        secret_value = input('Secret Value: ')

        self._data_client.set_secret(vault_info.properties.vault_uri, secret_name, secret_value)

        print('\nSecret %s added to vault %s' % (secret_name, vault_info.name))

    @staticmethod
    def _get_secret_name_from_url(url):
        split = url.split('/')
        return split[len(split) - 1]

    def _get_vault_list(self):
        vault_list = [vault for vault in self._mgmt_client.vaults.list()]

        return vault_list

config = KV_Config()

config.from_disk()

repl = KV_Repl(config)

repl.start()

config.to_disk()

print('success!')

<commit_msg>updates to kvrepl   -standardizing repl loops   -work around for get_secrets issue<commit_after>#!/usr/local/bin/python
from azure.common.credentials import ServicePrincipalCredentials
from azure.common.credentials import UserPassCredentials
from azure.mgmt.keyvault.models import Sku
from azure.mgmt.keyvault.models import VaultCreateOrUpdateParameters, VaultProperties, SkuName, AccessPolicyEntry, \
    Permissions, KeyPermissions, SecretPermissions, CertificatePermissions
from azure.common.credentials import BasicTokenAuthentication
from azure.keyvault import KeyVaultClient
from azure.mgmt.keyvault import KeyVaultManagementClient
from adal import token_cache
import adal
import json
import os
import sys

CLIENT_ID = '8fd4d3c4-efea-49aa-b1de-2c33c22da56e' # Azure cli
CLIENT_OID = '8694d835-b4e2-419a-a315-b13c854166e2'
CLIENT_TENANT_ID = 'a7fc734e-9961-43ce-b4de-21b8b38403ba'
KEY_VAULT_RESOURCE = 'https://vault.azure.net'
AZURE_MANAGEMENT_RESOURCE2 = 'https://management.core.windows.net/'

def _json_format(obj):
    return json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': '))

class KV_Config(dict):
    def __init__(self):

        self.authority_url = ''
        self.subscription_id = ''
        self.tenant_id = ''
        self.token_cache = ''
        self.user_id = ''
        self.resource_group = ''
        self.user_oid = ''
        self.location = 'westus'

    def __getattr__(self, item):
        return self[item]

    def __setattr__(self, key, value):
        self[key] = value

    def from_disk(self):
        if os.path.isfile('kvconfig.json'):
            with open('kvconfig.json', 'r') as configFile:
                try:
                    dict = json.load(configFile)
                except json.JSONDecodeError:
                    print('error loading config file')
                    return
                for key, value in dict.items():
                    if value:
                        self[key] = value

    def to_disk(self):
        with open('kvconfig.json', 'w') as configFile:
            json.dump(self, configFile, sort_keys=True, indent=4, separators=(',', ': '))

class KV_Auth(object):
    def __init__(self, config):
        self._config = config

        self._cache = token_cache.TokenCache()

        if self._config.token_cache:
            self._cache.deserialize(self._config.token_cache)

        self._context = adal.AuthenticationContext(self._config.authority_url, cache=self._cache)

    def get_keyvault_creds(self):
        return self._get_creds(KEY_VAULT_RESOURCE)

    def get_arm_creds(self):
        return self._get_creds(AZURE_MANAGEMENT_RESOURCE2)

    def _get_auth_token_from_code(self, resource):
        code = self._context.acquire_user_code(resource, CLIENT_ID)

        print(code['message'])

        token = self._context.acquire_token_with_device_code(resource, code, CLIENT_ID)

        return token

    def _get_creds(self, resource):
        token = None

        if not self._config.user_id:
            token = self._get_auth_token_from_code(resource)
        else:
            token = self._context.acquire_token(resource, self._config.user_id, CLIENT_ID)

            if not token:
                token = self._get_auth_token_from_code(resource)

        self._config.token_cache = self._cache.serialize()

        if token:
            if not self._config.user_id:
                self._config.user_id = token['userId']

            if not self._config.user_oid:
                self._config.user_oid = token['oid']

            token['access_token'] = token['accessToken']

            return BasicTokenAuthentication(token)

        return None

class KV_Repl(object):

    _repl_break_commands = set(('back', 'b'))

    _repl_quit_commands = set(('quit', 'q'))

    def __init__(self, config):
        self._auth = KV_Auth(config)
        self._config = config
        self._mgmt_client = KeyVaultManagementClient(self._auth.get_arm_creds(), config.subscription_id)
        self._data_client = KeyVaultClient(self._auth.get_keyvault_creds())
        self._selected_vault = None
        self._current_index = None

    def start(self):
        try:
            self._vault_index_loop();

        except SystemExit:
            print('\nuser exited\n')

    def _continue_repl(self, display_action, break_commands=()):
        display_action()

        self._selection = input('> ').lower()

        if self._selection in break_commands:
            return None

        elif self._selection in KV_Repl._repl_quit_commands:
            sys.exit()

        try:
            self._selection = int(self._selection)
        except ValueError:
            pass

        return self._selection

    def _display_vault_index(self):

        print('\nAvailable Vaults:\n')

        self._current_index = self._get_vault_list()

        for idx, vault in enumerate(self._current_index):
            print('%d. %s' % (idx, vault.name))

        print('\n#:select | (a)dd | (d)elete | (q)uit')

    def _vault_index_loop(self):
        while self._continue_repl(self._display_vault_index) is not None:
            vaults = self._current_index

            if isinstance(self._selection, int):
                i = self._selection

                if i >= 0 and i < len(vaults):
                    self._selected_vault = self._mgmt_client.vaults.get(self._config.resource_group, vaults[i].name)
                    self._vault_detail_loop()
                else:
                    print('invalid vault index')

            elif self._selection == 'a' or self._selection == 'add':
                self._add_vault()

            else:
                print('invalid input')

    def _add_vault(self):
        name = input('\nenter vault name:')

        all_perms = Permissions()
        all_perms.keys = [KeyPermissions.all]
        all_perms.secrets = [SecretPermissions.all]
        all_perms.certificates = [CertificatePermissions.all]

        user_policy = AccessPolicyEntry(self._config.tenant_id, self._config.user_oid, all_perms)

        app_policy = AccessPolicyEntry(CLIENT_TENANT_ID, CLIENT_OID, all_perms)

        access_policies = [user_policy, app_policy]

        properties = VaultProperties(self._config.tenant_id, Sku(name='standard'), access_policies)

        properties.enabled_for_deployment = True
        properties.enabled_for_disk_encryption = True
        properties.enabled_for_template_deployment = True

        vault = VaultCreateOrUpdateParameters(self._config.location, properties)

        self._mgmt_client.vaults.create_or_update(self._config.resource_group, name, vault)

        print('vault %s created\n' % name)

    def _display_selected_vault_detail(self):
        print('\nName:\t%s' % self._selected_vault.name)
        print('Uri:\t%s' % self._selected_vault.properties.vault_uri)
        print('Id:\t%s' % self._selected_vault.id)

        print('\n(s)ecrets (k)eys (c)ertificates (b)ack (q)uit\n')

    def _vault_detail_loop(self):

        while self._continue_repl(self._display_selected_vault_detail, break_commands=KV_Repl._repl_break_commands) is not None:

            if self._selection == 's' or self._selection == 'secrets':
                self._secret_index_loop()

            elif self._selection == 'k' or self._selection == 'keys':
                print('\nnot yet implemented\n')

            elif self._selection == 'c' or self._selection == 'certificates':
                print('\nnot yet implemented\n')

            else:
                print('invalid input')

    def _display_secret_index(self):
        self._current_index = []

        secret_iter = self._data_client.get_secrets(self._selected_vault.properties.vault_uri)

        if secret_iter is not None:
            try:
                self._current_index = [secret for secret in secret_iter]
            except TypeError:
                pass

        print('\n%s Secrets:\n' % self._selected_vault.name)

        for idx, s in enumerate(self._current_index):
            print('%d. %s' % (idx, KV_Repl._get_secret_name_from_url(s.id)))

        print('\n#:show secret value (a)dd (d)elete (b)ack (q)uit\n')

    def _secret_index_loop(self):

        while self._continue_repl(self._display_secret_index, break_commands=KV_Repl._repl_break_commands) is not None:

            secrets = self._current_index

            if isinstance(self._selection, int):
                i = self._selection

                if i >= 0 and i < len(secrets):
                    print('\n%s = %s\n' % (KV_Repl._get_secret_name_from_url(secrets[i].id), self._data_client.get_secret(secrets[i].id).value))
                else:
                    print('invalid secret index')

            elif self._selection == 'a' or self._selection == 'add':
                self._add_secret()

            elif self._selection == 'd' or self._selection == 'delete':
                print('\nnot yet implemented\n')

    def _add_secret(self):
        secret_name = input('\nSecret Name: ')
        secret_value = input('Secret Value: ')
        self._data_client.set_secret(self._selected_vault.properties.vault_uri, secret_name, secret_value)
        print('\nSecret %s added to vault %s' % (secret_name, self._selected_vault.name))

    @staticmethod
    def _get_secret_name_from_url(url):
        split = url.split('/')
        return split[len(split) - 1]

    def _get_vault_list(self):
        vault_list = [vault for vault in self._mgmt_client.vaults.list()]
        return vault_list

config = KV_Config()

config.from_disk()

repl = KV_Repl(config)

repl.start()

config.to_disk()

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import imp
import os
import random

from lightsweeper.lsgame import LSGameEngine
from lightsweeper.lsconfig import userSelect
from lightsweeper import lsconfig

conf = lsconfig.readConfiguration()
try:
    GAMESDIRS = [conf["GAMESDIR"]]
except KeyError:
    print("WARNING: 'GAMESDIR' is not set.")
    GAMESDIRS = [os.path.abspath(os.getcwd())]

NUMPLAYS = 0 # The number of games the player can play (0 is infinite/free play)

availableGames = dict()

for searchDir in GAMESDIRS:
    gameFiles = [os.path.join(os.getcwd(),f) for f in os.listdir(searchDir) if f.endswith(".py")]

    for gamePath in gameFiles:
        gameName = os.path.splitext(os.path.basename(gamePath))[0]
        fp, pathname, description = imp.find_module(gameName, GAMESDIRS)
        try:
            gameModule = imp.load_module(gameName, fp, pathname, description)
        finally:
            fp.close()
        try:
            exec("from {:s} import {:s} as thisGame".format(gameName, gameName))
        except ImportError as e:
            print("Warning: Unable to import {:s} ({:s})".format(gameName, e))
        availableGames[gameName] = thisGame

def clearTerm():
    os.system('cls' if os.name == 'nt' else 'clear')

def main():
    clearTerm()
    conf = lsconfig.LSFloorConfig()
    conf.selectConfig()
    while True:
        clearTerm()
        print(" L I G H T S W E E P E R ")

        games = [ g.__name__ for g in availableGames.values() ]

        games.append("Random")

        game = userSelect(games, "\nWhich game do you want?")

        if game is "Random":
            currentGame = list(availableGames.values())
        else:
            currentGame = availableGames[game]

        gameEngine = LSGameEngine(currentGame, conf.fileName)
        gameEngine.beginLoop(plays=NUMPLAYS)

if __name__ == '__main__':
    main()
<commit_msg>Sorting list of games<commit_after>#!/usr/bin/env python3

import imp
import os
import random

from lightsweeper.lsgame import LSGameEngine
from lightsweeper.lsconfig import userSelect
from lightsweeper import lsconfig

conf = lsconfig.readConfiguration()
try:
    GAMESDIRS = [conf["GAMESDIR"]]
except KeyError:
    print("WARNING: 'GAMESDIR' is not set.")
    GAMESDIRS = [os.path.abspath(os.getcwd())]

NUMPLAYS = 0 # The number of games the player can play (0 is infinite/free play)

availableGames = dict()

for searchDir in GAMESDIRS:
    gameFiles = [os.path.join(os.getcwd(),f) for f in os.listdir(searchDir) if f.endswith(".py")]

    for gamePath in gameFiles:
        gameName = os.path.splitext(os.path.basename(gamePath))[0]
        fp, pathname, description = imp.find_module(gameName, GAMESDIRS)
        try:
            gameModule = imp.load_module(gameName, fp, pathname, description)
        finally:
            fp.close()
        try:
            exec("from {:s} import {:s} as thisGame".format(gameName, gameName))
        except ImportError as e:
            print("Warning: Unable to import {:s} ({:s})".format(gameName, e))
        availableGames[gameName] = thisGame

def clearTerm():
    os.system('cls' if os.name == 'nt' else 'clear')

def main():
    clearTerm()
    conf = lsconfig.LSFloorConfig()
    conf.selectConfig()
    while True:
        clearTerm()
        print(" L I G H T S W E E P E R ")

        games = [ g.__name__ for g in availableGames.values() ]

        games.append("Random")

        game = userSelect(sorted(games), "\nWhich game do you want?")

        if game is "Random":
            currentGame = list(availableGames.values())
        else:
            currentGame = availableGames[game]

        gameEngine = LSGameEngine(currentGame, conf.fileName)
        gameEngine.beginLoop(plays=NUMPLAYS)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Create your views here.
from django.http import HttpResponse
from django.template import RequestContext, loader
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
import urllib
import urllib2
import requests
from django.shortcuts import redirect
from django.core.urlresolvers import reverse
from student.views import login_user
from student.views import _do_create_account
from student.views import activate_account
from mitxmako.shortcuts import render_to_response

from django.contrib.auth.models import User

from django.conf import settings

import re

from ims_lti_py import ToolConsumer, ToolConfig,\
        OutcomeRequest, OutcomeResponse
import hashlib
from student.views import course_from_id

from django.db import connection, connections, transaction

import time

from django.contrib.auth import authenticate
import django.contrib.auth

def login(request):
    '''
    If user is already logged in, redirects him to the dashboard
    If user isn't logged in, redirects them to WIND login
    If user is coming back from WIND, authenticates with ticket id
    If user is coming from the old CVN, authenticates with CVN-generated ticket
    '''
    reqData = request.POST
    if request.user.is_authenticated():
        return redirect(reverse('dashboard'))
    if 'ticketid' in request.GET:
        '''
        post_data = [('ticketid',request.GET.get('ticketid', '')),]     # a sequence of two element tuples
        result = urllib2.urlopen('http://400pixels.net/fakewind/fake_wind_validation.php?'+urllib.urlencode(post_data))
        content = result.read()
        return HttpResponse('Validation Successful!
Contents of ticket validation response:
'+content if 'yes' in content else 'Validation Failed!
Contents of ticket validation response:
'+content);
        #return HttpResponse("There's a GET message! ticketid is " +request.GET.get('ticketid', ""))
        '''
        user = authenticate(request=request, token=request.GET.get('ticketid', ''))

        if user is not None:
            if user.is_active:
                print "You provided a correct username and password!"
                django.contrib.auth.login(request, user)
            else:
                return HttpResponse('Your account has been disabled!')
        else:
            return HttpResponse('Your username and password were incorrect.')
        #if user is a professor, redirect them to cms
        user_groups = [g.name for g in user.groups.all()]
        pattern = re.compile("instructor|staff")
        returnable = ''
        for user_group in user_groups:
            if pattern.match(user_group):
                return redirect(settings.CMS_URL)
        return redirect(reverse('dashboard'))

    elif 'email' in reqData and 'first' in reqData and 'last' in reqData and 'token' in reqData:
        '''
        User is logging in via the old PHP CVN web app
        available GET variables: email, first, last, token
        '''
        user = authenticate(user_email=reqData['email'], first=reqData['first'],
                             last=reqData['last'], token=reqData['token'], username=None)
        if user is not None:
            if user.is_active:
                print "You provided a correct username and password!"
                django.contrib.auth.login(request, user)
            else:
                return HttpResponse('Your account has been disabled!')
        else:
            return HttpResponse('CVN authentication failed.')
        #if user is a professor, redirect them to cms
        user_groups = [g.name for g in user.groups.all()]
        pattern = re.compile("instructor|staff")
        returnable = ''
        for user_group in user_groups:
            if pattern.match(user_group):
                return redirect(settings.CMS_URL)
        return redirect(reverse('dashboard'))
    else:
        '''
        No post or get requests, so redirect user to Columbia WIND login
        '''
        return redirect(settings.WIND_LOGIN_URL + "?destination=" + settings.WIND_DESTINATION)
        '''template = loader.get_template('wind/index.html')
        context = RequestContext(request, {})
        return HttpResponse(template.render(context))'''

def fakewind(request):
    return HttpResponse("Hello, world. You're at fake WIND.")

def register(request):
    return redirect("http://cvn.columbia.edu/");

@login_required
@ensure_csrf_cookie
def course_dashboard(request, org, course, name):
    """
    Display an editable asset library

    org, course, name: Attributes of the Location for the item to edit
    """
    courseEnrollments = CourseEnrollment.objects.filter(course_id=org+'/'+course+'/'+name)
    '''
    returnable = ''
    for courseEnrollment in courseEnrollments:
        #user = User.objects.get(...
        returnable += str(courseEnrollment.user.username)+'
'
    return HttpResponse(returnable)
    return HttpResponse("Welcome to the professor dashboard!")
    '''

    return render_to_response('dashboard_index.html', {'courseEnrollments':courseEnrollments})
<commit_msg>Delete some commented code chunks<commit_after># Create your views here.
from django.http import HttpResponse
from django.template import RequestContext, loader
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
import requests
from django.shortcuts import redirect
from django.core.urlresolvers import reverse
from student.views import login_user
from student.views import _do_create_account
from student.views import activate_account
from mitxmako.shortcuts import render_to_response

from django.contrib.auth.models import User

from django.conf import settings

import re

from ims_lti_py import ToolConsumer, ToolConfig,\
        OutcomeRequest, OutcomeResponse
import hashlib
from student.views import course_from_id

from django.db import connection, connections, transaction

import time

from django.contrib.auth import authenticate
import django.contrib.auth

def login(request):
    '''
    If user is already logged in, redirects him to the dashboard
    If user isn't logged in, redirects them to WIND login
    If user is coming back from WIND, authenticates with ticket id
    If user is coming from the old CVN, authenticates with CVN-generated ticket
    '''
    reqData = request.POST
    if request.user.is_authenticated():
        return redirect(reverse('dashboard'))
    if 'ticketid' in request.GET:
        user = authenticate(request=request, token=request.GET.get('ticketid', ''))

        if user is not None:
            if user.is_active:
                print "You provided a correct username and password!"
                django.contrib.auth.login(request, user)
            else:
                return HttpResponse('Your account has been disabled!')
        else:
            return HttpResponse('Your username and password were incorrect.')
        #if user is a professor, redirect them to cms
        user_groups = [g.name for g in user.groups.all()]
        pattern = re.compile("instructor|staff")
        returnable = ''
        for user_group in user_groups:
            if pattern.match(user_group):
                return redirect(settings.CMS_URL)
        return redirect(reverse('dashboard'))

    elif 'email' in reqData and 'first' in reqData and 'last' in reqData and 'token' in reqData:
        '''
        User is logging in via the old PHP CVN web app
        available GET variables: email, first, last, token
        '''
        user = authenticate(user_email=reqData['email'], first=reqData['first'],
                             last=reqData['last'], token=reqData['token'], username=None)
        if user is not None:
            if user.is_active:
                print "You provided a correct username and password!"
                django.contrib.auth.login(request, user)
            else:
                return HttpResponse('Your account has been disabled!')
        else:
            return HttpResponse('CVN authentication failed.')
        #if user is a professor, redirect them to cms
        user_groups = [g.name for g in user.groups.all()]
        pattern = re.compile("instructor|staff")
        returnable = ''
        for user_group in user_groups:
            if pattern.match(user_group):
                return redirect(settings.CMS_URL)
        return redirect(reverse('dashboard'))
    else:
        '''
        No post or get requests, so redirect user to Columbia WIND login
        '''
        return redirect(settings.WIND_LOGIN_URL + "?destination=" + settings.WIND_DESTINATION)
        '''template = loader.get_template('wind/index.html')
        context = RequestContext(request, {})
        return HttpResponse(template.render(context))'''

def fakewind(request):
    return HttpResponse("Hello, world. You're at fake WIND.")

def register(request):
    return redirect("http://cvn.columbia.edu/");

@login_required
@ensure_csrf_cookie
def course_dashboard(request, org, course, name):
    """
    Display an editable asset library

    org, course, name: Attributes of the Location for the item to edit
    """
    courseEnrollments = CourseEnrollment.objects.filter(course_id=org+'/'+course+'/'+name)
    '''
    returnable = ''
    for courseEnrollment in courseEnrollments:
        #user = User.objects.get(...
        returnable += str(courseEnrollment.user.username)+'
'
    return HttpResponse(returnable)
    return HttpResponse("Welcome to the professor dashboard!")
    '''

    return render_to_response('dashboard_index.html', {'courseEnrollments':courseEnrollments})
<|endoftext|>


<commit_before>###############################################################
# RandomPlayer class by Daniel Field                          #
# refers to the lead sheet and timing to play random notes    #
# check out www.github.com/dan-field/jazz for info and rights #
###############################################################

from music import *
from random import *

class RandomPlayer:
   def __init__(self, ls=None):
      """Initialises a RandomPlayer object"""
      self.phr = Phrase()
      self.thisBeatRhythm = [0.5, 0.5]
      self.thisBeatNotes = [56, 57]
      self.thisBeatVelocities = [100, 100]
      self.nextBeatRhythm = [0.5, 0.5]
      self.nextBeatNotes = [58, 59]
      self.nextBeatVelocities = [100, 100]

   def playCurrentBeat(self, tempo=None):
      if tempo is None: tempo = 120
      self.phr.setTempo(tempo)
      self.thisBeatRhythm = self.nextBeatRhythm
      self.thisBeatNotes = self.nextBeatNotes
      self.thisBeatVelocities = self.nextBeatVelocities
      self.phr.addNoteList(self.thisBeatNotes, self.thisBeatRhythm, self.thisBeatVelocities)
      Play.midi(self.phr)
      self.phr.empty()

   def teeUpNextBeat(self):
      a = 0.5 #random()
      b = 1.0 - a
      c = randint(40, 80)
      d = c+randint(-5, 5)
      e = 100 + randint(-25, 25)
      f = e + randint(-20, 20)
      if f > 127: f=127
      self.nextBeatRhythm = [a, b]
      self.nextBeatNotes = [c, d]
      self.nextBeatVelocities = [e, f]

   def beat(self, tempo=None):
      self.playCurrentBeat(tempo)
      self.teeUpNextBeat()<commit_msg>Delete RandomPlayer.py<commit_after><|endoftext|>