<commit_before>from __future__ import absolute_import

import cgen as c
import numpy as np
from sympy import Indexed

from devito.cgen_utils import ccode
from devito.compiler import jit_compile
from devito.dimension import LoweredDimension
from devito.logger import yask as log, yask_warning as warning
from devito.ir.equations import LoweredEq
from devito.ir.iet import (Element, List, PointerCast, MetaCall, IsPerfectIteration,
                           Transformer, filter_iterations, retrieve_iteration_tree)
from devito.operator import OperatorRunnable
from devito.tools import flatten
from devito.types import Object

from devito.yask import nfac, exit, configuration
from devito.yask.data import DataScalar
from devito.yask.utils import (make_grid_accesses, make_sharedptr_funcall, rawpointer,
                               namespace)
from devito.yask.wrappers import YaskNullContext, YaskNullKernel, contexts
from devito.yask.types import YaskGridObject

__all__ = ['Operator']

class Operator(OperatorRunnable):

    """
    A special :class:`OperatorCore` to JIT-compile and run operators through YASK.
    """

    _default_headers = OperatorRunnable._default_headers
    _default_headers += ['#define restrict __restrict']
    _default_includes = OperatorRunnable._default_includes + ['yask_kernel_api.hpp']

    def __init__(self, expressions, **kwargs):
        super(Operator, self).__init__(expressions, **kwargs)
        # Each YASK Operator needs to have its own compiler (hence the copy()
        # below) because Operator-specific shared object will be added to the
        # list of linked libraries
        self._compiler = configuration.yask['compiler'].copy()

    def _specialize_exprs(self, expressions):
        expressions = super(Operator, self)._specialize_exprs(expressions)
        # No matter whether offloading will occur or not, all YASK grids accept
        # negative indices when using the get/set_element_* methods (up to the
        # padding extent), so the OOB-relative data space should be adjusted
        return [LoweredEq(e, e.ispace,
                          e.dspace.zero([d for d in e.dimensions if d.is_Space]),
                          e.reads, e.writes)
                for e in expressions]

    def _specialize_iet(self, iet):
        """Transform the Iteration/Expression tree to offload the computation of
        one or more loop nests onto YASK. This involves calling the YASK compiler
        to generate YASK code. Such YASK code is then called from within the
        transformed Iteration/Expression tree."""
        log("Specializing a Devito Operator for YASK...")

        self.context = YaskNullContext()
        self.yk_soln = YaskNullKernel()

        offloadable = find_offloadable_trees(iet)
        if len(offloadable) == 0:
            log("No offloadable trees found")
        elif len(offloadable) == 1:
            tree, grid, dtype = offloadable[0]
            self.context = contexts.fetch(grid, dtype)

            # Create a YASK compiler solution for this Operator
            yc_soln = self.context.make_yc_solution(namespace['jit-yc-soln'])

            transform = sympy2yask(self.context, yc_soln)
            try:
                for i in tree[-1].nodes:
                    transform(i.expr)

                funcall = make_sharedptr_funcall(namespace['code-soln-run'], ['time'],
                                                 namespace['code-soln-name'])
                funcall = Element(c.Statement(ccode(funcall)))
                iet = Transformer({tree[1]: funcall}).visit(iet)

                # Track /funcall/ as an external function call
                self.func_table[namespace['code-soln-run']] = MetaCall(None, False)

                # JIT-compile the newly-created YASK kernel
                local_grids = [i for i in transform.mapper if i.is_Array]
                self.yk_soln = self.context.make_yk_solution(namespace['jit-yk-soln'],
                                                             yc_soln, local_grids)

                # Print some useful information about the newly constructed solution
                log("Solution '%s' contains %d grid(s) and %d equation(s)." %
                    (yc_soln.get_name(), yc_soln.get_num_grids(),
                     yc_soln.get_num_equations()))

            except:
                log("Unable to offload a candidate tree.")
        else:
            exit("Found more than one offloadable trees in a single Operator")

        # Some Iteration/Expression trees are not offloaded to YASK and may
        # require further processing to be executed in YASK, due to the differences
        # in storage layout employed by Devito and YASK
        iet = make_grid_accesses(iet)

        log("Specialization successfully performed!")

        return iet

    def _build_parameters(self, iet):
        parameters = super(Operator, self)._build_parameters(iet)
        # Add parameters "disappeared" due to offloading
        parameters += tuple(i for i in self.input if i not in parameters)
        return parameters

    def _build_casts(self, iet):
        iet = super(Operator, self)._build_casts(iet)

        # Add YASK solution pointer for use in C-land
        soln_obj = Object(namespace['code-soln-name'], namespace['type-solution'])

        # Add YASK user and local grids pointers for use in C-land
        grid_objs = [YaskGridObject(i.name) for i in self.input if i.from_YASK]
        grid_objs.extend([YaskGridObject(i) for i in self.yk_soln.local_grids])

        # Build pointer casts
        casts = [PointerCast(soln_obj)] + [PointerCast(i) for i in grid_objs]

        return List(body=casts + [iet])

    def arguments(self, **kwargs):
        args = {}
        # Add in solution pointer
        args[namespace['code-soln-name']] = self.yk_soln.rawpointer
        # Add in local grids pointers
        for k, v in self.yk_soln.local_grids.items():
            args[namespace['code-grid-name'](k)] = rawpointer(v)
        return super(Operator, self).arguments(backend=args, **kwargs)

    def apply(self, **kwargs):
        # Build the arguments list to invoke the kernel function
        args = self.arguments(**kwargs)

        # Map default Functions to runtime Functions; will be used for "grid sharing"
        toshare = {}
        for i in self.input:
            v = kwargs.get(i.name, i)
            if np.isscalar(v):
                toshare[i] = DataScalar(v)
            elif i.from_YASK and (i.is_Constant or i.is_Function):
                toshare[v] = v.data

        log("Running YASK Operator through Devito...")
        arg_values = [args[p.name] for p in self.parameters]
        self.yk_soln.run(self.cfunction, arg_values, toshare)
        log("YASK Operator successfully run!")

        # Output summary of performance achieved
        return self._profile_output(args)

    @property
    def compile(self):
        """
        JIT-compile the C code generated by the Operator.

        It is ensured that JIT compilation will only be performed once per
        :class:`Operator`, reagardless of how many times this method is invoked.

        :returns: The file name of the JIT-compiled function.
        """
        if self._lib is None:
            # No need to recompile if a shared object has already been loaded.
            if not isinstance(self.yk_soln, YaskNullKernel):
                self._compiler.libraries.append(self.yk_soln.soname)
            return jit_compile(self.ccode, self._compiler)
        else:
            return self._lib.name

class sympy2yask(object):
    """
    Convert a SymPy expression into a YASK abstract syntax tree and create any
    necessay YASK grids.
    """

    def __init__(self, context, yc_soln):
        self.context = context
        self.yc_soln = yc_soln
        self.mapper = {}

    def __call__(self, expr):

        def nary2binary(args, op):
            r = run(args[0])
            return r if len(args) == 1 else op(r, nary2binary(args[1:], op))

        def run(expr):
            if expr.is_Integer:
                return nfac.new_const_number_node(int(expr))
            elif expr.is_Float:
                return nfac.new_const_number_node(float(expr))
            elif expr.is_Rational:
                a, b = expr.as_numer_denom()
                return nfac.new_const_number_node(float(a)/float(b))
            elif expr.is_Symbol:
                function = expr.base.function
                if function.is_Constant:
                    if function not in self.mapper:
                        self.mapper[function] = self.yc_soln.new_grid(function.name, [])
                    return self.mapper[function].new_relative_grid_point([])
                else:
                    # A DSE-generated temporary, which must have already been
                    # encountered as a LHS of a previous expression
                    assert function in self.mapper
                    return self.mapper[function]
            elif isinstance(expr, Indexed):
                function = expr.base.function
                if function not in self.mapper:
                    if function.is_TimeFunction:
                        dimensions = [nfac.new_step_index(function.indices[0].name)]
                        dimensions += [nfac.new_domain_index(i.name)
                                       for i in function.indices[1:]]
                    else:
                        dimensions = [nfac.new_domain_index(i.name)
                                      for i in function.indices]
                    self.mapper[function] = self.yc_soln.new_grid(function.name,
                                                                  dimensions)
                indices = [int((i.origin if isinstance(i, LoweredDimension) else i) - j)
                           for i, j in zip(expr.indices, function.indices)]
                return self.mapper[function].new_relative_grid_point(indices)
            elif expr.is_Add:
                return nary2binary(expr.args, nfac.new_add_node)
            elif expr.is_Mul:
                return nary2binary(expr.args, nfac.new_multiply_node)
            elif expr.is_Pow:
                base, exp = expr.as_base_exp()
                if not exp.is_integer:
                    warning("non-integer powers unsupported in Devito-YASK translation")
                    raise NotImplementedError

                if int(exp) < 0:
                    num, den = expr.as_numer_denom()
                    return nfac.new_divide_node(run(num), run(den))
                elif int(exp) >= 1:
                    return nary2binary([base] * exp, nfac.new_multiply_node)
                else:
                    warning("0-power found in Devito-YASK translation? setting to 1")
                    return nfac.new_const_number_node(1)
            elif expr.is_Equality:
                if expr.lhs.is_Symbol:
                    function = expr.lhs.base.function
                    assert function not in self.mapper
                    self.mapper[function] = run(expr.rhs)
                else:
                    return nfac.new_equation_node(*[run(i) for i in expr.args])
            else:
                warning("Missing handler in Devito-YASK translation")
                raise NotImplementedError

        return run(expr)

def find_offloadable_trees(iet):
    """
    Return the trees within ``iet`` that can be computed by YASK.

    A tree is "offloadable to YASK" if it is embedded in a time stepping loop
    *and* all of the grids accessed by the enclosed equations are homogeneous
    (i.e., same dimensions and data type).
    """
    offloadable = []
    for tree in retrieve_iteration_tree(iet):
        parallel = filter_iterations(tree, lambda i: i.is_Parallel)
        if not parallel:
            # Cannot offload non-parallel loops
            continue
        if not (IsPerfectIteration().visit(parallel[0]) and
                all(i.is_Expression for i in tree[-1].nodes)):
            # Don't know how to offload this Iteration/Expression to YASK
            continue
        functions = flatten(i.functions for i in tree[-1].nodes)
        keys = set((i.grid, i.dtype) for i in functions if i.is_TimeFunction)
        if len(keys) == 0:
            continue
        elif len(keys) > 1:
            exit("Cannot handle Operators w/ heterogeneous grids")
        grid, dtype = keys.pop()
        # Is this a "complete" tree iterating over the entire grid?
        dims = [i.dim for i in tree]
        if all(i in dims for i in grid.dimensions) and\
                any(j in dims for j in [grid.time_dim, grid.stepping_dim]):
            offloadable.append((tree, grid, dtype))
    return offloadable
<commit_msg>yask: Minor updates after changes to profiling<commit_after>from __future__ import absolute_import

import cgen as c
import numpy as np
from sympy import Indexed

from devito.cgen_utils import ccode
from devito.compiler import jit_compile
from devito.dimension import LoweredDimension
from devito.logger import yask as log, yask_warning as warning
from devito.ir.equations import LoweredEq
from devito.ir.iet import (Element, List, PointerCast, MetaCall, IsPerfectIteration,
                           Transformer, filter_iterations, retrieve_iteration_tree)
from devito.operator import OperatorRunnable
from devito.tools import flatten
from devito.types import Object

from devito.yask import nfac, exit, configuration
from devito.yask.data import DataScalar
from devito.yask.utils import (make_grid_accesses, make_sharedptr_funcall, rawpointer,
                               namespace)
from devito.yask.wrappers import YaskNullContext, YaskNullKernel, contexts
from devito.yask.types import YaskGridObject

__all__ = ['Operator']

class Operator(OperatorRunnable):

    """
    A special :class:`OperatorCore` to JIT-compile and run operators through YASK.
    """

    _default_headers = OperatorRunnable._default_headers
    _default_headers += ['#define restrict __restrict']
    _default_includes = OperatorRunnable._default_includes + ['yask_kernel_api.hpp']

    def __init__(self, expressions, **kwargs):
        super(Operator, self).__init__(expressions, **kwargs)
        # Each YASK Operator needs to have its own compiler (hence the copy()
        # below) because Operator-specific shared object will be added to the
        # list of linked libraries
        self._compiler = configuration.yask['compiler'].copy()

    def _specialize_exprs(self, expressions):
        expressions = super(Operator, self)._specialize_exprs(expressions)
        # No matter whether offloading will occur or not, all YASK grids accept
        # negative indices when using the get/set_element_* methods (up to the
        # padding extent), so the OOB-relative data space should be adjusted
        return [LoweredEq(e, e.ispace,
                          e.dspace.zero([d for d in e.dimensions if d.is_Space]),
                          e.reads, e.writes)
                for e in expressions]

    def _specialize_iet(self, iet):
        """Transform the Iteration/Expression tree to offload the computation of
        one or more loop nests onto YASK. This involves calling the YASK compiler
        to generate YASK code. Such YASK code is then called from within the
        transformed Iteration/Expression tree."""
        log("Specializing a Devito Operator for YASK...")

        self.context = YaskNullContext()
        self.yk_soln = YaskNullKernel()

        offloadable = find_offloadable_trees(iet)
        if len(offloadable) == 0:
            log("No offloadable trees found")
        elif len(offloadable) == 1:
            tree, bundle, grid, dtype = offloadable[0]
            self.context = contexts.fetch(grid, dtype)

            # Create a YASK compiler solution for this Operator
            yc_soln = self.context.make_yc_solution(namespace['jit-yc-soln'])

            transform = sympy2yask(self.context, yc_soln)
            try:
                for i in bundle.exprs:
                    transform(i.expr)

                funcall = make_sharedptr_funcall(namespace['code-soln-run'], ['time'],
                                                 namespace['code-soln-name'])
                funcall = Element(c.Statement(ccode(funcall)))
                iet = Transformer({tree[1]: funcall}).visit(iet)

                # Track /funcall/ as an external function call
                self.func_table[namespace['code-soln-run']] = MetaCall(None, False)

                # JIT-compile the newly-created YASK kernel
                local_grids = [i for i in transform.mapper if i.is_Array]
                self.yk_soln = self.context.make_yk_solution(namespace['jit-yk-soln'],
                                                             yc_soln, local_grids)

                # Print some useful information about the newly constructed solution
                log("Solution '%s' contains %d grid(s) and %d equation(s)." %
                    (yc_soln.get_name(), yc_soln.get_num_grids(),
                     yc_soln.get_num_equations()))

            except:
                log("Unable to offload a candidate tree.")
        else:
            exit("Found more than one offloadable trees in a single Operator")

        # Some Iteration/Expression trees are not offloaded to YASK and may
        # require further processing to be executed in YASK, due to the differences
        # in storage layout employed by Devito and YASK
        iet = make_grid_accesses(iet)

        log("Specialization successfully performed!")

        return iet

    def _build_parameters(self, iet):
        parameters = super(Operator, self)._build_parameters(iet)
        # Add parameters "disappeared" due to offloading
        parameters += tuple(i for i in self.input if i not in parameters)
        return parameters

    def _build_casts(self, iet):
        iet = super(Operator, self)._build_casts(iet)

        # Add YASK solution pointer for use in C-land
        soln_obj = Object(namespace['code-soln-name'], namespace['type-solution'])

        # Add YASK user and local grids pointers for use in C-land
        grid_objs = [YaskGridObject(i.name) for i in self.input if i.from_YASK]
        grid_objs.extend([YaskGridObject(i) for i in self.yk_soln.local_grids])

        # Build pointer casts
        casts = [PointerCast(soln_obj)] + [PointerCast(i) for i in grid_objs]

        return List(body=casts + [iet])

    def arguments(self, **kwargs):
        args = {}
        # Add in solution pointer
        args[namespace['code-soln-name']] = self.yk_soln.rawpointer
        # Add in local grids pointers
        for k, v in self.yk_soln.local_grids.items():
            args[namespace['code-grid-name'](k)] = rawpointer(v)
        return super(Operator, self).arguments(backend=args, **kwargs)

    def apply(self, **kwargs):
        # Build the arguments list to invoke the kernel function
        args = self.arguments(**kwargs)

        # Map default Functions to runtime Functions; will be used for "grid sharing"
        toshare = {}
        for i in self.input:
            v = kwargs.get(i.name, i)
            if np.isscalar(v):
                toshare[i] = DataScalar(v)
            elif i.from_YASK and (i.is_Constant or i.is_Function):
                toshare[v] = v.data

        log("Running YASK Operator through Devito...")
        arg_values = [args[p.name] for p in self.parameters]
        self.yk_soln.run(self.cfunction, arg_values, toshare)
        log("YASK Operator successfully run!")

        # Output summary of performance achieved
        return self._profile_output(args)

    @property
    def compile(self):
        """
        JIT-compile the C code generated by the Operator.

        It is ensured that JIT compilation will only be performed once per
        :class:`Operator`, reagardless of how many times this method is invoked.

        :returns: The file name of the JIT-compiled function.
        """
        if self._lib is None:
            # No need to recompile if a shared object has already been loaded.
            if not isinstance(self.yk_soln, YaskNullKernel):
                self._compiler.libraries.append(self.yk_soln.soname)
            return jit_compile(self.ccode, self._compiler)
        else:
            return self._lib.name

class sympy2yask(object):
    """
    Convert a SymPy expression into a YASK abstract syntax tree and create any
    necessay YASK grids.
    """

    def __init__(self, context, yc_soln):
        self.context = context
        self.yc_soln = yc_soln
        self.mapper = {}

    def __call__(self, expr):

        def nary2binary(args, op):
            r = run(args[0])
            return r if len(args) == 1 else op(r, nary2binary(args[1:], op))

        def run(expr):
            if expr.is_Integer:
                return nfac.new_const_number_node(int(expr))
            elif expr.is_Float:
                return nfac.new_const_number_node(float(expr))
            elif expr.is_Rational:
                a, b = expr.as_numer_denom()
                return nfac.new_const_number_node(float(a)/float(b))
            elif expr.is_Symbol:
                function = expr.base.function
                if function.is_Constant:
                    if function not in self.mapper:
                        self.mapper[function] = self.yc_soln.new_grid(function.name, [])
                    return self.mapper[function].new_relative_grid_point([])
                else:
                    # A DSE-generated temporary, which must have already been
                    # encountered as a LHS of a previous expression
                    assert function in self.mapper
                    return self.mapper[function]
            elif isinstance(expr, Indexed):
                function = expr.base.function
                if function not in self.mapper:
                    if function.is_TimeFunction:
                        dimensions = [nfac.new_step_index(function.indices[0].name)]
                        dimensions += [nfac.new_domain_index(i.name)
                                       for i in function.indices[1:]]
                    else:
                        dimensions = [nfac.new_domain_index(i.name)
                                      for i in function.indices]
                    self.mapper[function] = self.yc_soln.new_grid(function.name,
                                                                  dimensions)
                indices = [int((i.origin if isinstance(i, LoweredDimension) else i) - j)
                           for i, j in zip(expr.indices, function.indices)]
                return self.mapper[function].new_relative_grid_point(indices)
            elif expr.is_Add:
                return nary2binary(expr.args, nfac.new_add_node)
            elif expr.is_Mul:
                return nary2binary(expr.args, nfac.new_multiply_node)
            elif expr.is_Pow:
                base, exp = expr.as_base_exp()
                if not exp.is_integer:
                    warning("non-integer powers unsupported in Devito-YASK translation")
                    raise NotImplementedError

                if int(exp) < 0:
                    num, den = expr.as_numer_denom()
                    return nfac.new_divide_node(run(num), run(den))
                elif int(exp) >= 1:
                    return nary2binary([base] * exp, nfac.new_multiply_node)
                else:
                    warning("0-power found in Devito-YASK translation? setting to 1")
                    return nfac.new_const_number_node(1)
            elif expr.is_Equality:
                if expr.lhs.is_Symbol:
                    function = expr.lhs.base.function
                    assert function not in self.mapper
                    self.mapper[function] = run(expr.rhs)
                else:
                    return nfac.new_equation_node(*[run(i) for i in expr.args])
            else:
                warning("Missing handler in Devito-YASK translation")
                raise NotImplementedError

        return run(expr)

def find_offloadable_trees(iet):
    """
    Return the trees within ``iet`` that can be computed by YASK.

    A tree is "offloadable to YASK" if it is embedded in a time stepping loop
    *and* all of the grids accessed by the enclosed equations are homogeneous
    (i.e., same dimensions and data type).
    """
    offloadable = []
    for tree in retrieve_iteration_tree(iet):
        parallel = filter_iterations(tree, lambda i: i.is_Parallel)
        if not parallel:
            # Cannot offload non-parallel loops
            continue
        bundle = tree.inner.nodes[0]
        if len(tree.inner.nodes) > 1 or not bundle.is_ExpressionBundle:
            # Illegal nest
            continue
        if not IsPerfectIteration().visit(parallel[0]):
            # Don't know how to offload non-perfect nests
            continue
        functions = flatten(i.functions for i in bundle.exprs)
        keys = set((i.grid, i.dtype) for i in functions if i.is_TimeFunction)
        if len(keys) == 0:
            continue
        elif len(keys) > 1:
            exit("Cannot handle Operators w/ heterogeneous grids")
        grid, dtype = keys.pop()
        # Is this a "complete" tree iterating over the entire grid?
        dims = [i.dim for i in tree]
        if all(i in dims for i in grid.dimensions) and\
                any(j in dims for j in [grid.time_dim, grid.stepping_dim]):
            offloadable.append((tree, bundle, grid, dtype))
    return offloadable
<|endoftext|>


<commit_before>import os
import sys
import shutil
import time
import protoparser
import protoobjects
import utils
import minirest

def get_timestamp(): return time.strftime("%A, %d %b %Y %H:%M", time.localtime())

INDENT = "  "
def indent(count): return count * INDENT

def get_field_id(cmd_or_ev_name, recurse_list, field=""):
    if field:
        return "%s.%s.%s" % (cmd_or_ev_name, ".".join((m.name for m in recurse_list)), field.name)
    return "%s.%s" % (cmd_or_ev_name, ".".join((m.name for m in recurse_list)))

SOURCE_ROOT = os.path.dirname(os.path.abspath(__file__))
CSS_CLASSES = {protoobjects.NUMBER: "number",
               protoobjects.BUFFER: "string",
               protoobjects.BOOLEAN: "boolean"}
RESOURCES = "resources"
# index page
HEAD_INDEX = """<!doctype html>
<title>Scope Interface</title>
<link rel="stylesheet" href="./style/style.css">

Scope Interface STP/1

"""
SIDEPANEL_INDEX = """

All versions

%s

%s

"""
MAIN_INDEX = """

%s

All versions

%s

"""
H2_INDEX = """
%s %s
"""
H3_INDEX = """

%s %s

"""
H3_INDEX_ONLY_VERSION = """
 %s
"""
# service interface
HEAD = """<!doctype html>
<title>%s.%s</title>
<link rel="stylesheet" href="./style/style.css">

"""
H1 = """
%s%s</spnan>
\n"""
H2 = """
%s
\n"""
SIDEPANEL = """

back

Commands

%s

Events

%s

 <input type="checkbox" class="default-collapse"> collapse all by default

"""
SIDEPANLE_LINK = """
%s
\n"""
COMMAND = "".join(("
",
                   "",
                   "command",
                   " %s",
                   "(%s)",
                   "",
                   "
"))
RETURNS = "".join(("
",
                   "",
                   "returns",
                   " (%s)",
                   "",
                   "
"))
KEY = "".join(("
",
               "",
               "= %s;",
               "",
               "
"))
EVENT = "".join(("
",
                 "",
                 "event",
                 " %s",
                 " returns ",
                 "(%s)",
                 "",
                 "
"))
FIELD = "".join(("
",
                 "%s",
                 "",
                 "%s",
                 " %s",
                 " %s",
                 " = %s",
                 "%s",
                 ";",
                 "",
                 "
"))
ENUM = "".join(("
",
                "",
                "%s",
                " = ",
                "%s",
                ";",
                "",
                "
"))
EXPANDER = """ """

class ServiceDoc(object):
    def __init__(self, global_scope, protopath, dest):
        service = global_scope.service
        self.service = service
        self.protopath = protopath
        self.file_name = "%s.%s.html" % (service.name, service.version)
        self.html_path = os.path.join(dest, self.file_name)
        self.global_scope = global_scope

def copy_html_src(html_src, dest):
    for d in ["style", "script"]:
        dest_path = os.path.join(dest, d)
        if os.path.exists(dest_path): shutil.rmtree(dest_path)
        shutil.copytree(os.path.join(html_src, d), dest_path)

def print_doc(fp, obj, depth=0):
    if obj.doc:
        fp.write("
")
        fp.write(minirest.process(obj.doc_lines).serialize())
        fp.write("
")

def print_enum(fp, cmd_or_ev_name, enum, recurse_list):
    fp.write("
\n")
    fp.write("
{
\n")
    fp.write("
")
    if enum.doc:
        fp.write("
\n")
        print_doc(fp, enum)
        fp.write("
\n")
    for field in enum.fields:
        fp.write("
\n")
        field_id = get_field_id(cmd_or_ev_name, recurse_list, field)
        fp.write(ENUM % (field_id, field_id, field.name, field.key))
        print_doc(fp, field)
        fp.write("
\n")
    fp.write("
\n")
    fp.write("
}
\n")
    fp.write("
")

def print_message(fp, cmd_or_ev_name, msg, depth=0, recurse_list=[]):
    fp.write("
\n")
    fp.write("
{
\n")
    fp.write("
")
    if msg.doc:
        fp.write("
\n")
        print_doc(fp, msg, depth)
        fp.write("
\n")
    for field in msg.fields:
        field_id = get_field_id(cmd_or_ev_name, recurse_list, field)
        fp.write("
\n")
        f_type = field.type
        has_expander = (f_type.is_message and not f_type in recurse_list) or f_type.is_enum
        expander = EXPANDER if has_expander else ""
        css_class = CSS_CLASSES[f_type.sup_type] if f_type.sup_type in CSS_CLASSES else ""
        default_val = " [default = %s]" % field.default_value if field.default_value else ""
        args = field_id, expander, field_id, field.q, css_class, field.full_type_name, field.name, field.key, default_val
        fp.write(FIELD % args)
        print_doc(fp, field, depth)
        if f_type.is_message and not f_type in recurse_list:
            print_message(fp, cmd_or_ev_name, f_type, depth, recurse_list[:] + [field.type])
        if f_type.is_enum:
            print_enum(fp, cmd_or_ev_name, f_type, recurse_list)
        fp.write("
\n")
    fp.write("
\n")
    fp.write("
}
\n")
    fp.write("
")

def print_command(fp, command):
    fp.write(H2 % (command.name, command.name, command.name))
    if command.doc: print_doc(fp, command)
    recurse_list = [command.request_arg]
    field_id = get_field_id(command.name, recurse_list)
    fp.write(COMMAND % (field_id, field_id, command.name, command.request_arg.name))
    print_message(fp, command.name, command.request_arg, recurse_list=recurse_list)
    recurse_list = [command.response_arg]
    field_id = get_field_id(command.name, recurse_list)
    fp.write(RETURNS % (field_id, field_id, command.response_arg.name))
    print_message(fp, command.name, command.response_arg, recurse_list=recurse_list)
    field_id = "%s.key" % command.name
    fp.write(KEY % (field_id, field_id, command.key))

def print_event(fp, event):
    fp.write(H2 % (event.name, event.name, event.name))
    if event.doc: print_doc(fp, event)
    recurse_list = [event.response_arg]
    field_id = get_field_id(event.name, recurse_list)
    fp.write(EVENT % (field_id, field_id, event.name, event.response_arg.name))
    print_message(fp, event.name, event.response_arg, recurse_list=recurse_list)
    field_id = "%s.key" % event.name
    fp.write(KEY % (field_id, field_id, event.key))

def print_service(fp, services, service):
    fp.write(HEAD % (service.name, service.version))
    fp.write(H1 % (service.name, service.version))
    commands = service.command_names
    events = service.event_names
    command_links = (SIDEPANLE_LINK % (cmd, cmd) for cmd in commands)
    event_links = (SIDEPANLE_LINK % (ev, ev) for ev in events)
    fp.write(SIDEPANEL % ("".join(command_links), "".join(event_links)))
    fp.write("
\n")
    if service.doc: print_doc(fp, service)
    for cmd in commands: print_command(fp, getattr(service, cmd))
    for ev in events: print_event(fp, getattr(service, ev))
    fp.write("
\n")

def print_index(fp, services_dict, dest):
    fp.write(HEAD_INDEX)
    services = services_dict.items()
    services.sort(key= lambda s: s[0])
    all_versions_list = []
    latest_versions = []
    all_versions = []
    for name, versions_dict in services:
        all_versions_list.append(SIDEPANLE_LINK % (name, name))
        versions = versions_dict.values()
        versions.sort(key=lambda s: s.service.patch_version, reverse=True)
        versions.sort(key=lambda s: s.service.minor_version, reverse=True)
        versions.sort(key=lambda s: s.service.major_version, reverse=True)
        latest = versions[0]
        latest_file_name = "%s.html" % name
        latest_versions.append(H2_INDEX  % (latest_file_name, name, latest.service.version))
        shutil.copyfile(os.path.join(dest, latest.file_name), os.path.join(dest, latest_file_name))
        all_versions.append("
\n")
        for version in versions:
            if version == latest:
                args = name, version.file_name, name, version.service.version
                all_versions.append(H3_INDEX % args)
            else:
                args = version.file_name, version.service.version
                all_versions.append(H3_INDEX_ONLY_VERSION % args)
        all_versions.append("
\n")
    fp.write(SIDEPANEL_INDEX % ("".join(all_versions_list), get_timestamp()))
    fp.write(MAIN_INDEX % ("".join(latest_versions), "".join(all_versions)))

def get_scope_services(proto_paths, dest):
    services = {}
    for path in proto_paths:
        with open(path, "rb") as fp:
            g_scope = protoparser.parse(fp.read())
            service = services.setdefault(g_scope.service.name, {})
            service[g_scope.service.version] = ServiceDoc(g_scope, path, dest)
    return services

def scope_doc(args):
    if not os.path.exists(args.dest): os.mkdir(args.dest)
    proto_paths = [args.src] if os.path.isfile(args.src) else utils.get_proto_files(args.src)
    services = get_scope_services(proto_paths, args.dest)
    for service in services.values():
        for version in service.values():
            with open(version.html_path, "wb") as fp:
                print_service(fp, services, version.service)
    with open(os.path.join(args.dest, "index.html"), "wb") as fp:
         print_index(fp, services, args.dest)
    copy_html_src(os.path.join(SOURCE_ROOT, RESOURCES), args.dest)

def setup_subparser(subparsers, config):
    subp = subparsers.add_parser("scope-doc", help="Create scope interface documentation.")
    subp.add_argument("src", nargs="?", default=".", help="""proto file or directory (default: %(default)s)).""")
    subp.add_argument("dest", nargs="?",  default="msg-defs", help="the destination directory (default: %(default)s)).")
    subp.set_defaults(func=scope_doc)
<commit_msg>Added a link to Scope Architecure in the index.<commit_after>import os
import sys
import shutil
import time
import protoparser
import protoobjects
import utils
import minirest

def get_timestamp(): return time.strftime("%A, %d %b %Y %H:%M", time.localtime())

INDENT = "  "
def indent(count): return count * INDENT

def get_field_id(cmd_or_ev_name, recurse_list, field=""):
    if field:
        return "%s.%s.%s" % (cmd_or_ev_name, ".".join((m.name for m in recurse_list)), field.name)
    return "%s.%s" % (cmd_or_ev_name, ".".join((m.name for m in recurse_list)))

SOURCE_ROOT = os.path.dirname(os.path.abspath(__file__))
CSS_CLASSES = {protoobjects.NUMBER: "number",
               protoobjects.BUFFER: "string",
               protoobjects.BOOLEAN: "boolean"}
RESOURCES = "resources"
# index page
HEAD_INDEX = """<!doctype html>
<title>Scope Interface</title>
<link rel="stylesheet" href="./style/style.css">

Scope Interface STP/1

"""
SIDEPANEL_INDEX = """

Scope Architecure

All versions

%s

%s

"""
MAIN_INDEX = """

%s

All versions

%s

"""
H2_INDEX = """
%s %s
"""
H3_INDEX = """

%s %s

"""
H3_INDEX_ONLY_VERSION = """
 %s
"""
# service interface
HEAD = """<!doctype html>
<title>%s.%s</title>
<link rel="stylesheet" href="./style/style.css">

"""
H1 = """
%s%s</spnan>
\n"""
H2 = """
%s
\n"""
SIDEPANEL = """

back

Commands

%s

Events

%s

 <input type="checkbox" class="default-collapse"> collapse all by default

"""
SIDEPANLE_LINK = """
%s
\n"""
COMMAND = "".join(("
",
                   "",
                   "command",
                   " %s",
                   "(%s)",
                   "",
                   "
"))
RETURNS = "".join(("
",
                   "",
                   "returns",
                   " (%s)",
                   "",
                   "
"))
KEY = "".join(("
",
               "",
               "= %s;",
               "",
               "
"))
EVENT = "".join(("
",
                 "",
                 "event",
                 " %s",
                 " returns ",
                 "(%s)",
                 "",
                 "
"))
FIELD = "".join(("
",
                 "%s",
                 "",
                 "%s",
                 " %s",
                 " %s",
                 " = %s",
                 "%s",
                 ";",
                 "",
                 "
"))
ENUM = "".join(("
",
                "",
                "%s",
                " = ",
                "%s",
                ";",
                "",
                "
"))
EXPANDER = """ """

class ServiceDoc(object):
    def __init__(self, global_scope, protopath, dest):
        service = global_scope.service
        self.service = service
        self.protopath = protopath
        self.file_name = "%s.%s.html" % (service.name, service.version)
        self.html_path = os.path.join(dest, self.file_name)
        self.global_scope = global_scope

def copy_html_src(html_src, dest):
    for d in ["style", "script"]:
        dest_path = os.path.join(dest, d)
        if os.path.exists(dest_path): shutil.rmtree(dest_path)
        shutil.copytree(os.path.join(html_src, d), dest_path)

def print_doc(fp, obj, depth=0):
    if obj.doc:
        fp.write("
")
        fp.write(minirest.process(obj.doc_lines).serialize())
        fp.write("
")

def print_enum(fp, cmd_or_ev_name, enum, recurse_list):
    fp.write("
\n")
    fp.write("
{
\n")
    fp.write("
")
    if enum.doc:
        fp.write("
\n")
        print_doc(fp, enum)
        fp.write("
\n")
    for field in enum.fields:
        fp.write("
\n")
        field_id = get_field_id(cmd_or_ev_name, recurse_list, field)
        fp.write(ENUM % (field_id, field_id, field.name, field.key))
        print_doc(fp, field)
        fp.write("
\n")
    fp.write("
\n")
    fp.write("
}
\n")
    fp.write("
")

def print_message(fp, cmd_or_ev_name, msg, depth=0, recurse_list=[]):
    fp.write("
\n")
    fp.write("
{
\n")
    fp.write("
")
    if msg.doc:
        fp.write("
\n")
        print_doc(fp, msg, depth)
        fp.write("
\n")
    for field in msg.fields:
        field_id = get_field_id(cmd_or_ev_name, recurse_list, field)
        fp.write("
\n")
        f_type = field.type
        has_expander = (f_type.is_message and not f_type in recurse_list) or f_type.is_enum
        expander = EXPANDER if has_expander else ""
        css_class = CSS_CLASSES[f_type.sup_type] if f_type.sup_type in CSS_CLASSES else ""
        default_val = " [default = %s]" % field.default_value if field.default_value else ""
        args = field_id, expander, field_id, field.q, css_class, field.full_type_name, field.name, field.key, default_val
        fp.write(FIELD % args)
        print_doc(fp, field, depth)
        if f_type.is_message and not f_type in recurse_list:
            print_message(fp, cmd_or_ev_name, f_type, depth, recurse_list[:] + [field.type])
        if f_type.is_enum:
            print_enum(fp, cmd_or_ev_name, f_type, recurse_list)
        fp.write("
\n")
    fp.write("
\n")
    fp.write("
}
\n")
    fp.write("
")

def print_command(fp, command):
    fp.write(H2 % (command.name, command.name, command.name))
    if command.doc: print_doc(fp, command)
    recurse_list = [command.request_arg]
    field_id = get_field_id(command.name, recurse_list)
    fp.write(COMMAND % (field_id, field_id, command.name, command.request_arg.name))
    print_message(fp, command.name, command.request_arg, recurse_list=recurse_list)
    recurse_list = [command.response_arg]
    field_id = get_field_id(command.name, recurse_list)
    fp.write(RETURNS % (field_id, field_id, command.response_arg.name))
    print_message(fp, command.name, command.response_arg, recurse_list=recurse_list)
    field_id = "%s.key" % command.name
    fp.write(KEY % (field_id, field_id, command.key))

def print_event(fp, event):
    fp.write(H2 % (event.name, event.name, event.name))
    if event.doc: print_doc(fp, event)
    recurse_list = [event.response_arg]
    field_id = get_field_id(event.name, recurse_list)
    fp.write(EVENT % (field_id, field_id, event.name, event.response_arg.name))
    print_message(fp, event.name, event.response_arg, recurse_list=recurse_list)
    field_id = "%s.key" % event.name
    fp.write(KEY % (field_id, field_id, event.key))

def print_service(fp, services, service):
    fp.write(HEAD % (service.name, service.version))
    fp.write(H1 % (service.name, service.version))
    commands = service.command_names
    events = service.event_names
    command_links = (SIDEPANLE_LINK % (cmd, cmd) for cmd in commands)
    event_links = (SIDEPANLE_LINK % (ev, ev) for ev in events)
    fp.write(SIDEPANEL % ("".join(command_links), "".join(event_links)))
    fp.write("
\n")
    if service.doc: print_doc(fp, service)
    for cmd in commands: print_command(fp, getattr(service, cmd))
    for ev in events: print_event(fp, getattr(service, ev))
    fp.write("
\n")

def print_index(fp, services_dict, dest):
    fp.write(HEAD_INDEX)
    services = services_dict.items()
    services.sort(key= lambda s: s[0])
    all_versions_list = []
    latest_versions = []
    all_versions = []
    for name, versions_dict in services:
        all_versions_list.append(SIDEPANLE_LINK % (name, name))
        versions = versions_dict.values()
        versions.sort(key=lambda s: s.service.patch_version, reverse=True)
        versions.sort(key=lambda s: s.service.minor_version, reverse=True)
        versions.sort(key=lambda s: s.service.major_version, reverse=True)
        latest = versions[0]
        latest_file_name = "%s.html" % name
        latest_versions.append(H2_INDEX  % (latest_file_name, name, latest.service.version))
        shutil.copyfile(os.path.join(dest, latest.file_name), os.path.join(dest, latest_file_name))
        all_versions.append("
\n")
        for version in versions:
            if version == latest:
                args = name, version.file_name, name, version.service.version
                all_versions.append(H3_INDEX % args)
            else:
                args = version.file_name, version.service.version
                all_versions.append(H3_INDEX_ONLY_VERSION % args)
        all_versions.append("
\n")
    fp.write(SIDEPANEL_INDEX % ("".join(all_versions_list), get_timestamp()))
    fp.write(MAIN_INDEX % ("".join(latest_versions), "".join(all_versions)))

def get_scope_services(proto_paths, dest):
    services = {}
    for path in proto_paths:
        with open(path, "rb") as fp:
            g_scope = protoparser.parse(fp.read())
            service = services.setdefault(g_scope.service.name, {})
            service[g_scope.service.version] = ServiceDoc(g_scope, path, dest)
    return services

def scope_doc(args):
    if not os.path.exists(args.dest): os.mkdir(args.dest)
    proto_paths = [args.src] if os.path.isfile(args.src) else utils.get_proto_files(args.src)
    services = get_scope_services(proto_paths, args.dest)
    for service in services.values():
        for version in service.values():
            with open(version.html_path, "wb") as fp:
                print_service(fp, services, version.service)
    with open(os.path.join(args.dest, "index.html"), "wb") as fp:
         print_index(fp, services, args.dest)
    copy_html_src(os.path.join(SOURCE_ROOT, RESOURCES), args.dest)

def setup_subparser(subparsers, config):
    subp = subparsers.add_parser("scope-doc", help="Create scope interface documentation.")
    subp.add_argument("src", nargs="?", default=".", help="""proto file or directory (default: %(default)s)).""")
    subp.add_argument("dest", nargs="?",  default="msg-defs", help="the destination directory (default: %(default)s)).")
    subp.set_defaults(func=scope_doc)
<|endoftext|>


<commit_before># Copyright 2013, Red Hat Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from distutils import dir_util
import logging
import os
import string
import sys
import tempfile

from dib_elements.element import Element
from dib_elements.util import call

from diskimage_builder.elements import expand_dependencies

class ElementManager(object):

    def __init__(self, elements, hooks, element_paths=None, dry_run=False,
                 interactive=False):
        """
        :param elements: Element names to apply.
        :type elements: list.
        :param hooks: Hooks to run for each element.
        :type hooks: list.
        :param element_paths: File system paths to search for elements.
        :type element_paths: list of strings.
        :param dry_run: If True, do not actually run the hooks.
        :type dry_run: bool
        """
        self.elements = elements
        self.dry_run = dry_run
        self.hooks = hooks
        self.interactive = interactive
        self.loaded_elements = {}

        # the environment variable should override anything passed in
        if os.environ.has_key('ELEMENTS_PATH'):
            self.element_paths = os.environ['ELEMENTS_PATH'].split(':')
        else:
            self.element_paths = element_paths

        if self.element_paths is None:
            raise Exception

        logging.info('manager initialized with elements path: %s' %
                      self.element_paths)

        self.load_elements()
        self.load_dependencies()
        self.copy_elements()

    def run(self):
        """Apply the elements by running each specified hook."""
        for hook in self.hooks:
            logging.info("running hook: %s" % hook)
            self.run_hook(hook)

    def load_elements(self):
        """Load all elements from self.element_paths.

        This populates self.loaded_elements.
        """
        for path in self.element_paths:
            self.process_path(path)

    def copy_elements(self):
        """Copy elements to apply to a temporary directory."""
        self.tmp_hook_dir = tempfile.mkdtemp()
        for element in self.elements:
            element_dir = self.loaded_elements[element].directory
            dir_util.copy_tree(element_dir, self.tmp_hook_dir)
        # elements expect this environment variable to be set
        try:
            os.unlink('/tmp/in_target.d')
        except:
            pass
        os.link(self.tmp_hook_dir, '/tmp/in_target.d')
        os.environ['TMP_HOOKS_PATH'] = self.tmp_hook_dir
        os.environ['PATH'] = '%s:%s/bin' % (os.environ['PATH'],
                                            self.tmp_hook_dir)

    def process_path(self, path):
        """Load elements from a given filesystem path.

        :param path: Filesystem path from which to load elements.
        :type path: str.
        """
        if not os.access(path, os.R_OK):
            raise Exception

        for element in os.listdir(path):
            if not os.path.isdir(os.path.join(path, element)):
                continue
            if self.loaded_elements.has_key(element):
                raise Exception
            self.loaded_elements[element] = Element(os.path.join(path, element))

    def load_dependencies(self):
        """Load and add all element dependencies to self.elements."""
        all_elements = expand_dependencies(
            self.elements, ':'.join(self.element_paths))
        self.elements = all_elements

    def run_hook(self, hook):
        """Run a hook on the current system.

        TODO: update this to use dib-run-parts

        :param hook: name of hook to run
        :type hook: str
        """
        hook_dir = os.path.join(self.tmp_hook_dir, '%s.d' % hook)
        scripts = os.listdir(hook_dir)
        scripts = [s for s in scripts if s.startswith(tuple(string.digits))]
        scripts = sorted(scripts, key=lambda script: os.path.basename(script))
        scripts = [os.path.abspath(os.path.join(hook_dir, s)) for s in scripts]

        for script in scripts:
            logging.info("running script: %s" % script)
            if not self.dry_run:
                # environment must be preseverd so that the variables set
                # earlier in os.environ are available in the scripts.
                rc = call(['sudo', '-E', '/bin/bash', script])
                if rc != 0:
                    logging.error("scripted failed: %s" % script)
                    if self.interactive:
                        logging.error("Continue? (y/n): ")
                        sys.stdout.flush()
                        entry = raw_input("")
                        if entry.lower() == 'y':
                            logging.info("continuing on user command.")
                            continue
                    logging.error("exiting after failure.")
                    sys.exit(1)
            else:
                logging.info("script to execute: %s" % script)
<commit_msg>use symlink, not hard link<commit_after># Copyright 2013, Red Hat Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from distutils import dir_util
import logging
import os
import string
import sys
import tempfile

from dib_elements.element import Element
from dib_elements.util import call

from diskimage_builder.elements import expand_dependencies

class ElementManager(object):

    def __init__(self, elements, hooks, element_paths=None, dry_run=False,
                 interactive=False):
        """
        :param elements: Element names to apply.
        :type elements: list.
        :param hooks: Hooks to run for each element.
        :type hooks: list.
        :param element_paths: File system paths to search for elements.
        :type element_paths: list of strings.
        :param dry_run: If True, do not actually run the hooks.
        :type dry_run: bool
        """
        self.elements = elements
        self.dry_run = dry_run
        self.hooks = hooks
        self.interactive = interactive
        self.loaded_elements = {}

        # the environment variable should override anything passed in
        if os.environ.has_key('ELEMENTS_PATH'):
            self.element_paths = os.environ['ELEMENTS_PATH'].split(':')
        else:
            self.element_paths = element_paths

        if self.element_paths is None:
            raise Exception

        logging.info('manager initialized with elements path: %s' %
                      self.element_paths)

        self.load_elements()
        self.load_dependencies()
        self.copy_elements()

    def run(self):
        """Apply the elements by running each specified hook."""
        for hook in self.hooks:
            logging.info("running hook: %s" % hook)
            self.run_hook(hook)

    def load_elements(self):
        """Load all elements from self.element_paths.

        This populates self.loaded_elements.
        """
        for path in self.element_paths:
            self.process_path(path)

    def copy_elements(self):
        """Copy elements to apply to a temporary directory."""
        self.tmp_hook_dir = tempfile.mkdtemp()
        for element in self.elements:
            element_dir = self.loaded_elements[element].directory
            dir_util.copy_tree(element_dir, self.tmp_hook_dir)
        # elements expect this environment variable to be set
        try:
            os.unlink('/tmp/in_target.d')
        except:
            pass
        os.symlink(self.tmp_hook_dir, '/tmp/in_target.d')
        os.environ['TMP_HOOKS_PATH'] = self.tmp_hook_dir
        os.environ['PATH'] = '%s:%s/bin' % (os.environ['PATH'],
                                            self.tmp_hook_dir)

    def process_path(self, path):
        """Load elements from a given filesystem path.

        :param path: Filesystem path from which to load elements.
        :type path: str.
        """
        if not os.access(path, os.R_OK):
            raise Exception

        for element in os.listdir(path):
            if not os.path.isdir(os.path.join(path, element)):
                continue
            if self.loaded_elements.has_key(element):
                raise Exception
            self.loaded_elements[element] = Element(os.path.join(path, element))

    def load_dependencies(self):
        """Load and add all element dependencies to self.elements."""
        all_elements = expand_dependencies(
            self.elements, ':'.join(self.element_paths))
        self.elements = all_elements

    def run_hook(self, hook):
        """Run a hook on the current system.

        TODO: update this to use dib-run-parts

        :param hook: name of hook to run
        :type hook: str
        """
        hook_dir = os.path.join(self.tmp_hook_dir, '%s.d' % hook)
        scripts = os.listdir(hook_dir)
        scripts = [s for s in scripts if s.startswith(tuple(string.digits))]
        scripts = sorted(scripts, key=lambda script: os.path.basename(script))
        scripts = [os.path.abspath(os.path.join(hook_dir, s)) for s in scripts]

        for script in scripts:
            logging.info("running script: %s" % script)
            if not self.dry_run:
                # environment must be preseverd so that the variables set
                # earlier in os.environ are available in the scripts.
                rc = call(['sudo', '-E', '/bin/bash', script])
                if rc != 0:
                    logging.error("scripted failed: %s" % script)
                    if self.interactive:
                        logging.error("Continue? (y/n): ")
                        sys.stdout.flush()
                        entry = raw_input("")
                        if entry.lower() == 'y':
                            logging.info("continuing on user command.")
                            continue
                    logging.error("exiting after failure.")
                    sys.exit(1)
            else:
                logging.info("script to execute: %s" % script)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import importlib
import json
import sys

def loadAttributePlugins(attributes):
  for attribute in attributes:
    try:
      attribute['implementation'] = importlib.import_module("attributes.{0}.main".format(attribute['name']))
    except ImportError:
      print("Failed to load the {0} attribute.".format(attribute['name']))

def processConfiguration(config_file):
  try:
    config = json.load(config_file)
    return config
  except:
    print("Malformatted or missing configuration.")
    sys.exit(2)

def processArguments():
  parser = argparse.ArgumentParser(description='Calculate the score of a repository.')
  parser.add_argument('-c', '--config', type=argparse.FileType('r'), default='config.json', dest='config_file', help='Path to the configuration file.')
  parser.add_argument('repository_id', type=int, nargs=1, help='Identifier for a repository as it appears in the GHTorrent database.')

  if len(sys.argv) is 1:
    parser.print_help()
    sys.exit(1)

  return parser.parse_args()

def main():
  args = processArguments()
  config = processConfiguration(args.config_file)
  attributes = config['attributes']
  loadAttributePlugins(attributes)

  score = 0
  for attribute in attributes:
    result = attribute['implementation'].run(metadata, repo_path, attribute['options'])
    score += result * attribute['weight']

if __name__ == '__main__':
  main()
<commit_msg>Only load attribute if 'enabled'<commit_after>#!/usr/bin/env python3

import argparse
import importlib
import json
import sys

def loadAttributePlugins(attributes):
  for attribute in attributes:
    if attribute['enabled']:
      try:
        attribute['implementation'] = importlib.import_module("attributes.{0}.main".format(attribute['name']))
      except ImportError:
        print("Failed to load the {0} attribute.".format(attribute['name']))

def processConfiguration(config_file):
  try:
    config = json.load(config_file)
    return config
  except:
    print("Malformatted or missing configuration.")
    sys.exit(2)

def processArguments():
  parser = argparse.ArgumentParser(description='Calculate the score of a repository.')
  parser.add_argument('-c', '--config', type=argparse.FileType('r'), default='config.json', dest='config_file', help='Path to the configuration file.')
  parser.add_argument('repository_id', type=int, nargs=1, help='Identifier for a repository as it appears in the GHTorrent database.')

  if len(sys.argv) is 1:
    parser.print_help()
    sys.exit(1)

  return parser.parse_args()

def main():
  args = processArguments()
  config = processConfiguration(args.config_file)
  attributes = config['attributes']
  loadAttributePlugins(attributes)

  score = 0
  for attribute in attributes:
    result = attribute['implementation'].run(metadata, repo_path, attribute['options'])
    score += result * attribute['weight']

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: ascii -*-

# An init script for running Instant and a number of bots.

import os, re
import errno, signal
import shlex
import subprocess
import argparse

try: # Py3K
    from configparser import ConfigParser as _ConfigParser
except ImportError: # Py2K
    from ConfigParser import SafeConfigParser as _ConfigParser

DEFAULT_CONFFILE = 'config/instant.ini'
DEFAULT_PIDFILE_TEMPLATE = 'run/%s.pid'

PID_LINE_RE = re.compile(r'^[0-9]+\s*$')

class RunnerError(Exception): pass

class ConfigurationError(RunnerError): pass

class Process:
    def __init__(self, name, command, config=None):
        if config is None: config = {}
        self.name = name
        self.command = command
        self.pidfile = config.get('pidfile', DEFAULT_PIDFILE_TEMPLATE % name)
        self.workdir = config.get('workdir')
        self._pid = Ellipsis
        self._child = None
        self._prev_child = None

    def _read_pidfile(self):
        f = None
        try:
            f = open(self.pidfile)
            data = f.read()
            if not PID_LINE_RE.match(data):
                raise ValueError('Invalid PID file contents')
            ret = int(data)
            if ret < 0:
                raise ValueError('Invalid PID in PID file')
            return ret
        except IOError as e:
            if e.errno == errno.ENOENT: return None
            raise
        finally:
            if f: f.close()

    def _write_pidfile(self, pid):
        if pid is None:
            try:
                os.unlink(self.pidfile)
            except OSError as e:
                if e.errno == e.ENOENT: return
                raise
        else:
            with open(self.pidfile, 'w') as f:
                f.write('%s\n' % pid)

    def get_pid(self, force=False):
        if self._pid is Ellipsis or force:
            self._pid = self._read_pidfile()
        return self._pid

    def set_pid(self, pid):
        self._pid = pid
        self._write_pidfile(pid)

    def start(self):
        try:
            cur_status = self.status()
            if cur_status == 'RUNNING':
                return 'ALREADY_RUNNING'
        except IOError:
            pass
        self._child = subprocess.Popen(self.command, close_fds=False,
                                       cwd=self.workdir)
        self.set_pid(self._child.pid)
        return 'OK'

    def wait_start(self):
        return None

    def stop(self):
        if self._child is not None:
            self._child.terminate()
            self._prev_child = self._child
            self._child = None
        else:
            # We could theoretically record the PID for wait_stop(), but that
            # would be even more fragile than what we already do in status().
            self._prev_child = None
            pid = self.get_pid()
            if pid is None:
                return 'NOT_RUNNING'
            else:
                os.kill(pid, signal.SIGTERM)
        self.set_pid(None)
        return 'OK'

    def wait_stop(self):
        if self._prev_child:
            raw_status = self._prev_child.wait()
            status = 'OK %s' % (raw_status,)
            self._prev_child = None
        else:
            status = None
        return status

    def status(self):
        pid = self.get_pid()
        if pid is None:
            return 'NOT_RUNNING'
        try:
            os.kill(pid, 0)
            return 'RUNNING'
        except OSError as e:
            if e.errno == errno.ESRCH: return 'STALEFILE'
            raise

class ProcessGroup:
    def __init__(self):
        self.processes = []

    def add(self, proc):
        self.processes.append(proc)

    def _for_each(self, handler, verbose=True):
        for p in self.processes:
            try:
                result = handler(p)
            except Exception as exc:
                result = 'ERROR (%s: %s)' % (type(exc).__name__, exc)
            if verbose:
                if result is None: result = 'NA'
                print ('%s: %s' % (p.name, result))

    def start(self, verbose=False):
        self._for_each(lambda p: p.start(), verbose)

    def wait_start(self, verbose=False):
        self._for_each(lambda p: p.wait_start(), verbose)

    def stop(self, verbose=False):
        self._for_each(lambda p: p.stop(), verbose)

    def wait_stop(self, verbose=False):
        self._for_each(lambda p: p.wait_stop(), verbose)

    def status(self, verbose=False):
        self._for_each(lambda p: p.status(), verbose)

class InstantManager(ProcessGroup):
    def __init__(self, conffile=None):
        ProcessGroup.__init__(self)
        if conffile is None: conffile = DEFAULT_CONFFILE
        self.conffile = conffile
        self.config = None

    def init(self):
        self.config = _ConfigParser()
        self.config.read(self.conffile)
        sections = [s for s in self.config.sections()
                    if s == 'instant' or s.startswith('scribe-')]
        sections.sort()
        for s in sections:
            values = dict(self.config.items(s))
            try:
                cmdline = values['cmdline']
            except KeyError:
                raise ConfigurationError('Missing required key "cmdline" in '
                    'section "%s"' % s)
            command = tuple(shlex.split(cmdline))
            self.add(Process(s, command, values))

    def dispatch(self, cmd):
        try:
            func = {'start': self.do_start, 'stop': self.do_stop,
                    'status': self.do_status}[cmd]
        except KeyError:
            raise RunnerError('Unknown command: ' + cmd)
        func()

    def do_start(self):
        self.start(verbose=True)

    def do_stop(self):
        self.stop(verbose=False)

    def do_status(self):
        self.status(verbose=True)

def main():
    p = argparse.ArgumentParser(
        description='Manage an Instant backend and a group of bots')
    p.add_argument('--config', '-c', default=DEFAULT_CONFFILE,
                   help='Configuration file location (default %(default)s)')
    sp = p.add_subparsers(dest='cmd', description='The action to perform')
    sp.add_parser('start', help='Start the backend and bots')
    sp.add_parser('stop', help='Stop the backend and bots')
    sp.add_parser('status', help='Check whether backend or bots are running')
    arguments = p.parse_args()
    mgr = InstantManager(arguments.config)
    try:
        mgr.init()
    except ConfigurationError as exc:
        raise SystemExit('Configuration error: ' + str(exc))
    mgr.dispatch(arguments.cmd)

if __name__ == '__main__': main()
<commit_msg>[Scripts] Add redirection framework to run.py<commit_after>#!/usr/bin/env python3
# -*- coding: ascii -*-

# An init script for running Instant and a number of bots.

import os, re
import errno, signal
import shlex
import subprocess
import argparse

try: # Py3K
    from configparser import ConfigParser as _ConfigParser
except ImportError: # Py2K
    from ConfigParser import SafeConfigParser as _ConfigParser

DEFAULT_CONFFILE = 'config/instant.ini'
DEFAULT_PIDFILE_TEMPLATE = 'run/%s.pid'

REDIRECTION_RE = re.compile(r'^[<>|&]+')
PID_LINE_RE = re.compile(r'^[0-9]+\s*$')

class RunnerError(Exception): pass

class ConfigurationError(RunnerError): pass

class Redirection:
    @classmethod
    def parse(cls, text):
        text = text.strip()
        try:
            constant = {'': None,
                        'stdout': subprocess.STDOUT,
                        'devnull': subprocess.DEVNULL}[text]
        except KeyError:
            pass
        else:
            return cls(constant)
        m = REDIRECTION_RE.match(text)
        if not m:
            raise ValueError('Invalid redirection string: %r' % (text,))
        prefix, filename = m.group(0), text[m.end():].strip()
        try:
            mode = {'<': 'rb', '>': 'wb', '>>': 'ab', '<>': 'r+'}[prefix]
        except KeyError:
            raise ValueError('Invalid redirection operator: %s' % prefix)
        return cls(filename, mode)

    def __init__(self, target, mode=None):
        self.target = target
        self.mode = mode

    def open(self):
        if isinstance(self.target, str):
            return open(self.target, self.mode)
        else:
            return self.target

    def close(self, fp):
        if hasattr(fp, 'close'):
            try:
                fp.close()
            except Exception:
                pass

class Process:
    def __init__(self, name, command, config=None):
        if config is None: config = {}
        self.name = name
        self.command = command
        self.pidfile = config.get('pidfile', DEFAULT_PIDFILE_TEMPLATE % name)
        self.workdir = config.get('workdir')
        self._pid = Ellipsis
        self._child = None
        self._prev_child = None

    def _read_pidfile(self):
        f = None
        try:
            f = open(self.pidfile)
            data = f.read()
            if not PID_LINE_RE.match(data):
                raise ValueError('Invalid PID file contents')
            ret = int(data)
            if ret < 0:
                raise ValueError('Invalid PID in PID file')
            return ret
        except IOError as e:
            if e.errno == errno.ENOENT: return None
            raise
        finally:
            if f: f.close()

    def _write_pidfile(self, pid):
        if pid is None:
            try:
                os.unlink(self.pidfile)
            except OSError as e:
                if e.errno == e.ENOENT: return
                raise
        else:
            with open(self.pidfile, 'w') as f:
                f.write('%s\n' % pid)

    def get_pid(self, force=False):
        if self._pid is Ellipsis or force:
            self._pid = self._read_pidfile()
        return self._pid

    def set_pid(self, pid):
        self._pid = pid
        self._write_pidfile(pid)

    def start(self):
        try:
            cur_status = self.status()
            if cur_status == 'RUNNING':
                return 'ALREADY_RUNNING'
        except IOError:
            pass
        self._child = subprocess.Popen(self.command, close_fds=False,
                                       cwd=self.workdir)
        self.set_pid(self._child.pid)
        return 'OK'

    def wait_start(self):
        return None

    def stop(self):
        if self._child is not None:
            self._child.terminate()
            self._prev_child = self._child
            self._child = None
        else:
            # We could theoretically record the PID for wait_stop(), but that
            # would be even more fragile than what we already do in status().
            self._prev_child = None
            pid = self.get_pid()
            if pid is None:
                return 'NOT_RUNNING'
            else:
                os.kill(pid, signal.SIGTERM)
        self.set_pid(None)
        return 'OK'

    def wait_stop(self):
        if self._prev_child:
            raw_status = self._prev_child.wait()
            status = 'OK %s' % (raw_status,)
            self._prev_child = None
        else:
            status = None
        return status

    def status(self):
        pid = self.get_pid()
        if pid is None:
            return 'NOT_RUNNING'
        try:
            os.kill(pid, 0)
            return 'RUNNING'
        except OSError as e:
            if e.errno == errno.ESRCH: return 'STALEFILE'
            raise

class ProcessGroup:
    def __init__(self):
        self.processes = []

    def add(self, proc):
        self.processes.append(proc)

    def _for_each(self, handler, verbose=True):
        for p in self.processes:
            try:
                result = handler(p)
            except Exception as exc:
                result = 'ERROR (%s: %s)' % (type(exc).__name__, exc)
            if verbose:
                if result is None: result = 'NA'
                print ('%s: %s' % (p.name, result))

    def start(self, verbose=False):
        self._for_each(lambda p: p.start(), verbose)

    def wait_start(self, verbose=False):
        self._for_each(lambda p: p.wait_start(), verbose)

    def stop(self, verbose=False):
        self._for_each(lambda p: p.stop(), verbose)

    def wait_stop(self, verbose=False):
        self._for_each(lambda p: p.wait_stop(), verbose)

    def status(self, verbose=False):
        self._for_each(lambda p: p.status(), verbose)

class InstantManager(ProcessGroup):
    def __init__(self, conffile=None):
        ProcessGroup.__init__(self)
        if conffile is None: conffile = DEFAULT_CONFFILE
        self.conffile = conffile
        self.config = None

    def init(self):
        self.config = _ConfigParser()
        self.config.read(self.conffile)
        sections = [s for s in self.config.sections()
                    if s == 'instant' or s.startswith('scribe-')]
        sections.sort()
        for s in sections:
            values = dict(self.config.items(s))
            try:
                cmdline = values['cmdline']
            except KeyError:
                raise ConfigurationError('Missing required key "cmdline" in '
                    'section "%s"' % s)
            command = tuple(shlex.split(cmdline))
            self.add(Process(s, command, values))

    def dispatch(self, cmd):
        try:
            func = {'start': self.do_start, 'stop': self.do_stop,
                    'status': self.do_status}[cmd]
        except KeyError:
            raise RunnerError('Unknown command: ' + cmd)
        func()

    def do_start(self):
        self.start(verbose=True)

    def do_stop(self):
        self.stop(verbose=False)

    def do_status(self):
        self.status(verbose=True)

def main():
    p = argparse.ArgumentParser(
        description='Manage an Instant backend and a group of bots')
    p.add_argument('--config', '-c', default=DEFAULT_CONFFILE,
                   help='Configuration file location (default %(default)s)')
    sp = p.add_subparsers(dest='cmd', description='The action to perform')
    sp.add_parser('start', help='Start the backend and bots')
    sp.add_parser('stop', help='Stop the backend and bots')
    sp.add_parser('status', help='Check whether backend or bots are running')
    arguments = p.parse_args()
    mgr = InstantManager(arguments.config)
    try:
        mgr.init()
    except ConfigurationError as exc:
        raise SystemExit('Configuration error: ' + str(exc))
    mgr.dispatch(arguments.cmd)

if __name__ == '__main__': main()
<|endoftext|>


<commit_before>"""
freesurfer tools
"""

import os
import re
import nibabel as nib
import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
import nipype.interfaces.freesurfer as fs
import nipype.interfaces.utility as util
import nipype.interfaces.fsl as fsl

def reconall(subjfile,subjID=None,subjdir=None):
    """
    Carries out Freesurfer's reconall on T1 nifti file

    WARNING: Reconall takes very long to run!!

    http://nipy.sourceforge.net/nipype/users/examples/smri_freesurfer.html

    Parameters
    ----------
    subjfile: nifti file
        Path to subject's T1 nifti file

    subjID: string
        optional name for subject's output folder

    subjdir: the directory to where segmentation results should be saved. Defaults to same directory as subjfile.
    """

    T1dir = os.path.dirname(subjfile)
    filename = os.path.basename(subjfile)

    # Tell freesurfer what subjects directory to use
    if subjdir==None:
        subjdir=T1dir
    fs.FSCommand.set_default_subjects_dir(subjdir)
    segdir=subjdir+'/'+subjID+'/'
    print 'saving to ' + subjdir

    # subject ID
    if subjID==None:
        m=re.search('(\w+?)_*_',subjfile)
        subjID=m.group(0) + 'seg'

    # check if file exists
    if os.path.isfile(subjfile):
        print 'running recon-all on ' + filename
    else:
        raise ValueError("File: %s does not exist!"%filename)

    # check if nifti format
    ext = os.path.splitext(filename)[-1].lower()
    if ext != ".nii":
        raise ValueError("File: %s is not a nifti file!"%filename)

    wf = pe.Workflow(name="segment")
    wf.base_dir = T1dir

    # run recon-all
    reconall = pe.Node(interface=fs.ReconAll(), name='reconall')
    reconall.inputs.subject_id = subjID
    reconall.inputs.directive = 'all'
    reconall.inputs.subjects_dir = subjdir
    reconall.inputs.T1_files = subjfile

    wf.add_nodes([reconall])
    result = wf.run()

    wf2 = pe.Workflow(name="convertmgz")
    wf2.base_dir = T1dir

    # convert ribbon.mgz to nii
    convertmgz = pe.Node(interface=fs.MRIConvert(), name='convertmgz')
    convertmgz.inputs.in_file = segdir+'mri/ribbon.mgz'
    convertmgz.inputs.out_orientation='LPS'
    convertmgz.inputs.resample_type= 'nearest'
    convertmgz.inputs.reslice_like= subjfile
    convertmgz.inputs.out_file=segdir+subjID+'_gmwm.nii.gz'

    wf2.add_nodes([convertmgz])
    result2 = wf2.run()
    return (result, result2)

def MRSvoxelStats(segfile, center, dim=[25.0,25.0,25.0], subjID=None):
    """
    returns grey/white/CSF content within MRS voxel

    Parameters
    ----------
    segfile:  nifti file
        path to segmentation file with grey/white matter labels.

    center : integer array
        [x,y,z] where x, y and z are the coordinates of the point of interest

    dim : float array
        dimensions of voxel in mm

    subjID: string
        optional subject identifier. Defaults to nims scan number

    See http://miykael.github.io/nipype-beginner-s-guide/regionOfInterest.html

    """
    # subject ID
    if subjID==None:
        m=re.search('(\w+?)_*_',segfile)
        subjID=m.group(0)[:-1]

    # get segmentation file
    gmwm = nib.load(segfile)
    gmwm_data = gmwm.get_data().squeeze()
    segdir = os.path.dirname(segfile)

    # calculate beginning corner of MRS voxel
    print 'Creating mask with center '+str(center[0])+', '+str(center[1])+', '+str(center[2])+', dimensions '+str(dim[0]) + ', '+str(dim[1]) +', '+str(dim[2]) +'mm.'

    corner = [center[0]-dim[0]/2,
              center[1]-dim[1]/2,
              center[2]-dim[2]/2]

    ROImask = pe.Node(interface=fsl.ImageMaths(),name="ROImask")
    ROIValues = (corner[0],dim[0],corner[1],dim[1],corner[2],dim[2])
    ROImask.inputs.op_string = '-mul 0 -add 1 -roi %d %d %d %d %d %d 0 1'%ROIValues
    ROImask.inputs.out_data_type = 'float'
    ROImask.inputs.in_file=segfile
    ROImask.inputs.out_file=segdir+subjID+'_ROIMask.nii.gz'

    mask_wf=pe.Workflow(name="ROImask")
    mask_wf.add_nodes([ROImask])
    mask_wf.run()

    # multiply voxel ROI with segfile
    roifile=segdir+subjID+'_ROIMask.nii.gz'
    roi = nib.load(roifile)
    roi_data = roi.get_data().squeeze()

    masked=np.multiply(roi_data,gmwm_data)

    # extract stats from a given segmentation
    total = size(nonzero(roi_data==1))
    white = size(nonzero(masked==42)) + size(nonzero(masked==3))
    grey = size(nonzero(masked==41)) + size(nonzero(masked==2))
    other = total - white - grey

    # proportions
    pWhite = white/total
    pGrey = grey/total
    pOther = other/total

    return (total, grey, white, other, pGrey, pWhite, pOther)

<commit_msg>fix proportion calculation<commit_after>"""
freesurfer tools
"""

import os
import re
import numpy as np
import nibabel as nib
import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
import nipype.interfaces.freesurfer as fs
import nipype.interfaces.utility as util
import nipype.interfaces.fsl as fsl

def reconall(subjfile,subjID=None,subjdir=None):
    """
    Carries out Freesurfer's reconall on T1 nifti file

    WARNING: Reconall takes very long to run!!

    http://nipy.sourceforge.net/nipype/users/examples/smri_freesurfer.html

    Parameters
    ----------
    subjfile: nifti file
        Path to subject's T1 nifti file

    subjID: string
        optional name for subject's output folder

    subjdir: the directory to where segmentation results should be saved. Defaults to same directory as subjfile.
    """

    T1dir = os.path.dirname(subjfile)
    filename = os.path.basename(subjfile)

    # Tell freesurfer what subjects directory to use
    if subjdir==None:
        subjdir=T1dir
    fs.FSCommand.set_default_subjects_dir(subjdir)
    segdir=subjdir+'/'+subjID+'/'
    print 'saving to ' + subjdir

    # subject ID
    if subjID==None:
        m=re.search('(\w+?)_*_',subjfile)
        subjID=m.group(0) + 'seg'

    # check if file exists
    if os.path.isfile(subjfile):
        print 'running recon-all on ' + filename
    else:
        raise ValueError("File: %s does not exist!"%filename)

    # check if nifti format
    ext = os.path.splitext(filename)[-1].lower()
    if ext != ".nii":
        raise ValueError("File: %s is not a nifti file!"%filename)

    wf = pe.Workflow(name="segment")
    wf.base_dir = T1dir

    # run recon-all
    reconall = pe.Node(interface=fs.ReconAll(), name='reconall')
    reconall.inputs.subject_id = subjID
    reconall.inputs.directive = 'all'
    reconall.inputs.subjects_dir = subjdir
    reconall.inputs.T1_files = subjfile

    wf.add_nodes([reconall])
    result = wf.run()

    wf2 = pe.Workflow(name="convertmgz")
    wf2.base_dir = T1dir

    # convert ribbon.mgz to nii
    convertmgz = pe.Node(interface=fs.MRIConvert(), name='convertmgz')
    convertmgz.inputs.in_file = segdir+'mri/ribbon.mgz'
    convertmgz.inputs.out_orientation='LPS'
    convertmgz.inputs.resample_type= 'nearest'
    convertmgz.inputs.reslice_like= subjfile
    convertmgz.inputs.out_file=segdir+subjID+'_gmwm.nii.gz'

    wf2.add_nodes([convertmgz])
    result2 = wf2.run()
    return (result, result2)

def MRSvoxelStats(segfile, center, dim=[25.0,25.0,25.0], subjID=None):
    """
    returns grey/white/CSF content within MRS voxel

    Parameters
    ----------
    segfile:  nifti file
        path to segmentation file with grey/white matter labels.

    center : integer array
        [x,y,z] where x, y and z are the coordinates of the point of interest

    dim : float array
        dimensions of voxel in mm

    subjID: string
        optional subject identifier. Defaults to nims scan number

    See http://miykael.github.io/nipype-beginner-s-guide/regionOfInterest.html

    """
    # subject ID
    if subjID==None:
        m=re.search('(\w+?)_*_',segfile)
        subjID=m.group(0)[:-1]

    # get segmentation file
    gmwm = nib.load(segfile)
    gmwm_data = gmwm.get_data().squeeze()
    segdir = os.path.dirname(segfile)

    # calculate beginning corner of MRS voxel
    print 'Creating mask with center '+str(center[0])+', '+str(center[1])+', '+str(center[2])+', dimensions '+str(dim[0]) + ', '+str(dim[1]) +', '+str(dim[2]) +'mm.'

    corner = [center[0]-dim[0]/2,
              center[1]-dim[1]/2,
              center[2]-dim[2]/2]

    ROImask = pe.Node(interface=fsl.ImageMaths(),name="ROImask")
    ROIValues = (corner[0],dim[0],corner[1],dim[1],corner[2],dim[2])
    ROImask.inputs.op_string = '-mul 0 -add 1 -roi %d %d %d %d %d %d 0 1'%ROIValues
    ROImask.inputs.out_data_type = 'float'
    ROImask.inputs.in_file=segfile
    ROImask.inputs.out_file=segdir+subjID+'_ROIMask.nii.gz'

    mask_wf=pe.Workflow(name="ROImask")
    mask_wf.add_nodes([ROImask])
    mask_wf.run()

    # multiply voxel ROI with segfile
    roifile=segdir+subjID+'_ROIMask.nii.gz'
    roi = nib.load(roifile)
    roi_data = roi.get_data().squeeze()

    masked=np.multiply(roi_data,gmwm_data)

    # extract stats from a given segmentation
    total = np.size(np.nonzero(roi_data==1))
    white = np.size(np.nonzero(masked==42)) + np.size(np.nonzero(masked==3))
    grey = np.size(np.nonzero(masked==41)) + np.size(np.nonzero(masked==2))
    other = total - white - grey

    # proportions
    pWhite = 1.0*white/total
    pGrey = 1.0*grey/total
    pOther = 1.0*other/total

    return (total, grey, white, other, pGrey, pWhite, pOther)

<|endoftext|>


<commit_before>import socket
import time
import thread
import os
import BaseHTTPServer
import SimpleHTTPServer

class Data:
	data = "require('update')([lat,lon]);"

def collectData():
	global dataObj;
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #Create a TCPIP socket
	s.bind(("0.0.0.0", 8787)) #Bind to all interfaces on port 8787
	s.listen(100) #Accept a maximum of 100 connection simultaneously
	while True:
		c = s.accept()	#Accept a new connection
		thread.start_new_thread( setDataWithGPS, (c,)) #"Bind" to a GPS

def setDataWithGPS(sock):
	(client,addr) = sock;
	global dataObj
	while True:
		dataObj.data = client.recv(64).decode("utf-8");
		print dataObj.data;
		time.sleep(5);

class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
	def do_GET(self):
		print "CALLED!";
		global dataObj
		self.send_response(200)
            	self.send_header('Content-type','text/html')
                self.send_header('Access-Control-Allow-Origin', '*')
           	self.end_headers()
		self.wfile.write(dataObj.data)

dataObj = Data();	#Initiialize global var
thread.start_new_thread( collectData, ())	#Begin accepting data from GPS

server_address = ("0.0.0.0", 8789)
server = BaseHTTPServer.HTTPServer(server_address, RequestHandler)
server.serve_forever()		#Start HTTP server to serve clients
<commit_msg>Tiny error<commit_after>import socket
import time
import thread
import os
import BaseHTTPServer
import SimpleHTTPServer

class Data:
	data = "require('update')([null,null]);"

def collectData():
	global dataObj;
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #Create a TCPIP socket
	s.bind(("0.0.0.0", 8787)) #Bind to all interfaces on port 8787
	s.listen(100) #Accept a maximum of 100 connection simultaneously
	while True:
		c = s.accept()	#Accept a new connection
		thread.start_new_thread( setDataWithGPS, (c,)) #"Bind" to a GPS

def setDataWithGPS(sock):
	(client,addr) = sock;
	global dataObj
	while True:
		dataObj.data = client.recv(64).decode("utf-8");
		print dataObj.data;
		time.sleep(5);

class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
	def do_GET(self):
		print "CALLED!";
		global dataObj
		self.send_response(200)
            	self.send_header('Content-type','text/html')
                self.send_header('Access-Control-Allow-Origin', '*')
           	self.end_headers()
		self.wfile.write(dataObj.data)

dataObj = Data();	#Initiialize global var
thread.start_new_thread( collectData, ())	#Begin accepting data from GPS

server_address = ("0.0.0.0", 8789)
server = BaseHTTPServer.HTTPServer(server_address, RequestHandler)
server.serve_forever()		#Start HTTP server to serve clients
<|endoftext|>