<commit_before>import os
import numpy as np

import nengo
import nengo.builder
from pacman103.lib import graph, data_spec_gen, lib_map, parameters
from pacman103.front.common import enums

class EnsembleVertex(graph.Vertex):
    """PACMAN Vertex for an Ensemble."""

    REGIONS = enums.enum1(
        'SYSTEM',
        'BIAS',
        'ENCODERS',
        'DECODERS',
        'OUTPUT_KEYS'
    )

    def __init__(self, ens, constraints=None):
        """Create a new EnsembleVertex using the given Ensemble to generate
        appropriate parameters.

        :param ens: Ensemble to represent in the simulation.
        """
        # Save a reference to the ensemble before unpacking some useful values
        self._ens = ens

        # Create random number generator
        if ens.seed is None:
            rng = np.random.RandomState(rng.tomaxint())
        else:
            rng = np.random.RandomState(ens.seed)
        self.rng = rng

        # Generate eval points
        if ens.eval_points is None:
            # TODO: standardize how to set number of samples
            #  (this is different than the reference implementation!)
            S = min(ens.dimensions * 500, 5000)
            self.eval_points = nengo.decoders.sample_hypersphere(
                ens.dimensions, S, rng) * ens.radius
        else:
            self.eval_points = np.array(ens.eval_points, dtype=np.float64)
            if self.eval_points.ndim == 1:
                self.eval_points.shape = (-1, 1)

        # TODO: change this to not modify Model
        # Set up neurons
        if ens.neurons.gain is None or ens.neurons.bias is None:
            # if max_rates and intercepts are distributions,
            # turn them into fixed samples.
            if hasattr(ens.max_rates, 'sample'):
                ens.max_rates = ens.max_rates.sample(
                    ens.neurons.n_neurons, rng=rng
                )
            if hasattr(ens.intercepts, 'sample'):
                ens.intercepts = ens.intercepts.sample(
                    ens.neurons.n_neurons, rng=rng
                )
            ens.neurons.set_gain_bias(ens.max_rates, ens.intercepts)

        self.bias = ens.neurons.bias
        self.gain = ens.neurons.gain
        self.tau_rc = ens.neurons.tau_rc
        self.tau_ref = ens.neurons.tau_ref

        self.n_input_dimensions = ens.dimensions

        # Set up encoders
        if ens.encoders is None:
            self.encoders = ens.neurons.default_encoders(ens.dimensions, rng)
        else:
            self.encoders = np.array(ens.encoders, dtype=np.float64)
            enc_shape = (ens.neurons.n_neurons, ens.dimensions)
            if self.encoders.shape != enc_shape:
                raise nengo.builder.ShapeMismatch(
                    "Encoder shape is %s. Should be (n_neurons, dimensions);"
                    " in this case %s." % (self.encoders.shape, enc_shape)
                )

            norm = np.sum(self.encoders ** 2, axis=1)[:, np.newaxis]
            self.encoders /= np.sqrt(norm)
        ens.encoders = self.encoders   # TODO: remove this when it is no longer
                                       # required be Ensemble.activities()

        # For constant value injection
        self.direct_input = np.zeros(self._ens.dimensions)

        # Create the vertex
        super(EnsembleVertex, self).__init__(
            self._ens.n_neurons, constraints=constraints, label=ens.label
        )

    @property
    def _tau_ref_in_steps(self):
        # TODO: Check this
        return self.tau_ref * 10**-3

    @property
    def _one_over_tau_rc(self):
        return 1. / self.tau_rc

    def sizeof_region_system(self):
        """Get the size (in bytes) of the SYSTEM region."""
        # 8 words, 4 bytes per word
        return 4 * 8

    def sizeof_region_bias(self, n_atoms):
        """Get the size (in bytes) of the BIAS region for the given number of
        neurons/atoms.
        """
        # 1 word per atom
        return 4 * n_atoms

    def sizeof_region_encoders(self, n_atoms):
        """Get the size (in bytes) of the ENCODERS region for the given number
        of neurons/atoms.
        """
        # 1 word per atom per input dimension
        return 4 * n_atoms * self.n_input_dimensions

    def sizeof_region_decoders(self, n_atoms):
        """Get the size (in bytes) of the DECODERS region for the given number
        of neurons/atoms.
        """
        # 1 word per atom per output dimension
        return 4 * n_atoms * self.n_output_dimensions

    def sizeof_region_output_keys(self):
        """Get the size (in bytes) of the OUTPUT_KEYS region."""
        # 1 word per output dimension
        return 4 * self.n_output_dimensions

    # FOR UPSTREAM CHANGES
    def sdram_usage(self, lo_atom, hi_atom):
        """Return the amount of SDRAM used for the specified atoms."""
        # At the moment this is the same as the DTCM usage, though this may
        # change.
        return self.dtcm_usage(lo_atom, hi_atom)

    # FOR UPSTREAM CHANGES
    def dtcm_usage(self, lo_atom, hi_atom):
        """Return the amount of DTCM used for the specified atoms."""
        n_atoms = hi_atom - lo_atom
        return sum([
            self.sizeof_region_system(),
            self.sizeof_region_bias(n_atoms),
            self.sizeof_region_encoders(n_atoms),
            self.sizeof_region_decoders(n_atoms),
            self.sizeof_region_output_keys(),
        ])

    # FOR UPSTREAM CHANGES
    def cpu_usage(self, lo_atom, hi_atom):
        """Return the CPU utilisation for the specified atoms."""
        # TODO: Calculate this
        return 0

    # FOR UPSTREAM CHANGES
    def get_resources_for_atoms(self, lo_atom, hi_atom, n_machine_time_steps,
                                machine_time_step_us, partition_data_object):
        """Get the resources required for the specified atoms.

        :param lo_atom: Index of the lowest atom to represent
        :param hi_atom: Index of the highest atom to represent
        :param n_machine_time_steps: Duration of the simulation
        :param machine_time_step_us: Duration of a machine time step in us
        :param partition_data_object: ?

        :returns: A tuple of the partition data object, and the resources
                  required.
        """
        return (
            partition_data_object,
            lib_map.Resources(
                self.cpu_usage(lo_atom, hi_atom),
                self.dtcm_usage(lo_atom, hi_atom),
                self.sdram_usage(lo_atom, hi_atom)
            )
        )

    # TO BE DEPRECATED
    def get_requirements_per_atom(self):
        """Return some indication of the cost of including a single atom on a
        processing core.
        """
        return lib_map.Resources(
            1,
            self.dtcm_usage(0, self.atoms) / self.atoms,
            self.sdram_usage(0, self.atoms) / self.atoms
        )

    def generateDataSpec(self, processor, subvertex, dao):
        """Generate the data spec for the given subvertex."""
        # Create a spec for the subvertex
        subvertex.spec = data_spec_gen.DataSpec(processor, dao)
        subvertex.spec.initialise(0xABCD, dao)
        subvertex.spec.comment("# Nengo Ensemble")

        # Finalise the values for this Ensemble
        # Encode any constant inputs, and add to the biases
        self.bias = np.dot(self.encoders, self.direct_input)

        # Fill in the spec
        self.reserve_regions(subvertex)
        self.write_region_system(subvertex)
        self.write_region_bias(subvertex)
        self.write_region_encoders(subvertex)
        self.write_region_decoders(subvertex)
        self.write_region_output_keys(subvertex)

        # Get the executable
        x, y, p = processor.get_coordinates()
        executable_target = lib_map.ExecutableTarget(
            os.path.join(dao.get_binaries_directory(), 'nengo_ensemble.aplx'),
            x, y, p
        )

        return (executable_target, list(), list())

    def reserve_regions(self, subvertex):
        """Reserve sufficient space for the regions in the spec."""
        for (region, sizeof) in [
            (self.REGIONS.SYSTEM, self.sizeof_region_system),
            (self.REGIONS.BIAS, self.sizeof_region_bias),
            (self.REGIONS.ENCODERS, self.sizeof_region_encoders),
            (self.REGIONS.DECODERS, self.sizeof_region_decoders),
            (self.REGIONS.OUTPUT_KEYS, self.sizeof_region_output_keys),
        ]:
            subvertex.spec.reserveMemRegion(region, sizeof(subvertex.n_atoms))

    def write_region_system(self, subvertex):
        """Write the system region for the given subvertex."""
        subvertex.spec.comment("""# System Region
        # -------------
        # 1. Number of input dimensions
        # 2. Number of output dimensions
        # 3. Number of neurons
        # 4. Machine time step in us
        # 5. tau_ref in number of steps
        # 6. tau_rc ^ -1
        # 7. Filter decay (TO BE CHANGED)
        # 8. 1 - Filter decay (TO BE CHANGED)
        """)
        subvertex.spec.write(data=self.n_input_dimensions)
        subvertex.spec.write(data=self.n_output_dimensions)
        subvertex.spec.write(data=subvertex.n_atoms)
        subvertex.spec.write(data=int(0.001 * 10**-6))
        subvertex.spec.write(data=self._tau_ref_in_steps)
        subvertex.spec.write(data=parameters.s1615(self._one_over_tau_rc))
        # subvertex.spec.write(data=... FILTER DECAY ...)
        # subvertex.spec.write(data=... FILTER DECAY COMPLEMENT ...)

    def write_region_bias(self, subvertex):
        """Write the bias region for the given subvertex."""
        for n in range(subvertex.lo_atom, subvertex.hi_atom + 1):
            # Write the bias for all atoms within this subvertex
            subvertex.spec.write(data=parameters.s1615(self.bias[n]))

    def write_region_encoders(self, subvertex):
        """Write the encoder region for the given subvertex."""
        for n in range(subvertex.lo_atom, subvertex.hi_atom + 1):
            for d in range(self.data.n_input_dimensions):
                subvertex.spec.write(
                    data=parameters.s1615(
                        self.data.encoders[n, d] * self.data.gain[n]
                    )
                )

    def write_region_decoders(self, subvertex):
        """Write the decoder region for the given subvertex."""
        raise NotImplementedError

    def write_region_output_keys(self, subvertex):
        """Write the output keys region for the given subvertex."""
        raise NotImplementedError
<commit_msg>Bugfix: Switch between regions.<commit_after>import os
import numpy as np

import nengo
import nengo.builder
from pacman103.lib import graph, data_spec_gen, lib_map, parameters
from pacman103.front.common import enums

class EnsembleVertex(graph.Vertex):
    """PACMAN Vertex for an Ensemble."""

    REGIONS = enums.enum1(
        'SYSTEM',
        'BIAS',
        'ENCODERS',
        'DECODERS',
        'OUTPUT_KEYS'
    )

    def __init__(self, ens, constraints=None):
        """Create a new EnsembleVertex using the given Ensemble to generate
        appropriate parameters.

        :param ens: Ensemble to represent in the simulation.
        """
        # Save a reference to the ensemble before unpacking some useful values
        self._ens = ens

        # Create random number generator
        if ens.seed is None:
            rng = np.random.RandomState(rng.tomaxint())
        else:
            rng = np.random.RandomState(ens.seed)
        self.rng = rng

        # Generate eval points
        if ens.eval_points is None:
            # TODO: standardize how to set number of samples
            #  (this is different than the reference implementation!)
            S = min(ens.dimensions * 500, 5000)
            self.eval_points = nengo.decoders.sample_hypersphere(
                ens.dimensions, S, rng) * ens.radius
        else:
            self.eval_points = np.array(ens.eval_points, dtype=np.float64)
            if self.eval_points.ndim == 1:
                self.eval_points.shape = (-1, 1)

        # TODO: change this to not modify Model
        # Set up neurons
        if ens.neurons.gain is None or ens.neurons.bias is None:
            # if max_rates and intercepts are distributions,
            # turn them into fixed samples.
            if hasattr(ens.max_rates, 'sample'):
                ens.max_rates = ens.max_rates.sample(
                    ens.neurons.n_neurons, rng=rng
                )
            if hasattr(ens.intercepts, 'sample'):
                ens.intercepts = ens.intercepts.sample(
                    ens.neurons.n_neurons, rng=rng
                )
            ens.neurons.set_gain_bias(ens.max_rates, ens.intercepts)

        self.bias = ens.neurons.bias
        self.gain = ens.neurons.gain
        self.tau_rc = ens.neurons.tau_rc
        self.tau_ref = ens.neurons.tau_ref

        self.n_input_dimensions = ens.dimensions

        # Set up encoders
        if ens.encoders is None:
            self.encoders = ens.neurons.default_encoders(ens.dimensions, rng)
        else:
            self.encoders = np.array(ens.encoders, dtype=np.float64)
            enc_shape = (ens.neurons.n_neurons, ens.dimensions)
            if self.encoders.shape != enc_shape:
                raise nengo.builder.ShapeMismatch(
                    "Encoder shape is %s. Should be (n_neurons, dimensions);"
                    " in this case %s." % (self.encoders.shape, enc_shape)
                )

            norm = np.sum(self.encoders ** 2, axis=1)[:, np.newaxis]
            self.encoders /= np.sqrt(norm)
        ens.encoders = self.encoders   # TODO: remove this when it is no longer
                                       # required be Ensemble.activities()

        # For constant value injection
        self.direct_input = np.zeros(self._ens.dimensions)

        # Create the vertex
        super(EnsembleVertex, self).__init__(
            self._ens.n_neurons, constraints=constraints, label=ens.label
        )

    @property
    def _tau_ref_in_steps(self):
        # TODO: Check this
        return self.tau_ref * 10**-3

    @property
    def _one_over_tau_rc(self):
        return 1. / self.tau_rc

    def sizeof_region_system(self):
        """Get the size (in bytes) of the SYSTEM region."""
        # 8 words, 4 bytes per word
        return 4 * 8

    def sizeof_region_bias(self, n_atoms):
        """Get the size (in bytes) of the BIAS region for the given number of
        neurons/atoms.
        """
        # 1 word per atom
        return 4 * n_atoms

    def sizeof_region_encoders(self, n_atoms):
        """Get the size (in bytes) of the ENCODERS region for the given number
        of neurons/atoms.
        """
        # 1 word per atom per input dimension
        return 4 * n_atoms * self.n_input_dimensions

    def sizeof_region_decoders(self, n_atoms):
        """Get the size (in bytes) of the DECODERS region for the given number
        of neurons/atoms.
        """
        # 1 word per atom per output dimension
        return 4 * n_atoms * self.n_output_dimensions

    def sizeof_region_output_keys(self):
        """Get the size (in bytes) of the OUTPUT_KEYS region."""
        # 1 word per output dimension
        return 4 * self.n_output_dimensions

    # FOR UPSTREAM CHANGES
    def sdram_usage(self, lo_atom, hi_atom):
        """Return the amount of SDRAM used for the specified atoms."""
        # At the moment this is the same as the DTCM usage, though this may
        # change.
        return self.dtcm_usage(lo_atom, hi_atom)

    # FOR UPSTREAM CHANGES
    def dtcm_usage(self, lo_atom, hi_atom):
        """Return the amount of DTCM used for the specified atoms."""
        n_atoms = hi_atom - lo_atom
        return sum([
            self.sizeof_region_system(),
            self.sizeof_region_bias(n_atoms),
            self.sizeof_region_encoders(n_atoms),
            self.sizeof_region_decoders(n_atoms),
            self.sizeof_region_output_keys(),
        ])

    # FOR UPSTREAM CHANGES
    def cpu_usage(self, lo_atom, hi_atom):
        """Return the CPU utilisation for the specified atoms."""
        # TODO: Calculate this
        return 0

    # FOR UPSTREAM CHANGES
    def get_resources_for_atoms(self, lo_atom, hi_atom, n_machine_time_steps,
                                machine_time_step_us, partition_data_object):
        """Get the resources required for the specified atoms.

        :param lo_atom: Index of the lowest atom to represent
        :param hi_atom: Index of the highest atom to represent
        :param n_machine_time_steps: Duration of the simulation
        :param machine_time_step_us: Duration of a machine time step in us
        :param partition_data_object: ?

        :returns: A tuple of the partition data object, and the resources
                  required.
        """
        return (
            partition_data_object,
            lib_map.Resources(
                self.cpu_usage(lo_atom, hi_atom),
                self.dtcm_usage(lo_atom, hi_atom),
                self.sdram_usage(lo_atom, hi_atom)
            )
        )

    # TO BE DEPRECATED
    def get_requirements_per_atom(self):
        """Return some indication of the cost of including a single atom on a
        processing core.
        """
        return lib_map.Resources(
            1,
            self.dtcm_usage(0, self.atoms) / self.atoms,
            self.sdram_usage(0, self.atoms) / self.atoms
        )

    def generateDataSpec(self, processor, subvertex, dao):
        """Generate the data spec for the given subvertex."""
        # Create a spec for the subvertex
        subvertex.spec = data_spec_gen.DataSpec(processor, dao)
        subvertex.spec.initialise(0xABCD, dao)
        subvertex.spec.comment("# Nengo Ensemble")

        # Finalise the values for this Ensemble
        # Encode any constant inputs, and add to the biases
        self.bias = np.dot(self.encoders, self.direct_input)

        # Fill in the spec
        self.reserve_regions(subvertex)
        self.write_region_system(subvertex)
        self.write_region_bias(subvertex)
        self.write_region_encoders(subvertex)
        self.write_region_decoders(subvertex)
        self.write_region_output_keys(subvertex)

        # Get the executable
        x, y, p = processor.get_coordinates()
        executable_target = lib_map.ExecutableTarget(
            os.path.join(dao.get_binaries_directory(), 'nengo_ensemble.aplx'),
            x, y, p
        )

        return (executable_target, list(), list())

    def reserve_regions(self, subvertex):
        """Reserve sufficient space for the regions in the spec."""
        for (region, sizeof) in [
            (self.REGIONS.SYSTEM, self.sizeof_region_system),
            (self.REGIONS.BIAS, self.sizeof_region_bias),
            (self.REGIONS.ENCODERS, self.sizeof_region_encoders),
            (self.REGIONS.DECODERS, self.sizeof_region_decoders),
            (self.REGIONS.OUTPUT_KEYS, self.sizeof_region_output_keys),
        ]:
            subvertex.spec.reserveMemRegion(region, sizeof(subvertex.n_atoms))

    def write_region_system(self, subvertex):
        """Write the system region for the given subvertex."""
        subvertex.spec.switchWriteFocus(REGIONS.SYSTEM)
        subvertex.spec.comment("""# System Region
        # -------------
        # 1. Number of input dimensions
        # 2. Number of output dimensions
        # 3. Number of neurons
        # 4. Machine time step in us
        # 5. tau_ref in number of steps
        # 6. tau_rc ^ -1
        # 7. Filter decay (TO BE CHANGED)
        # 8. 1 - Filter decay (TO BE CHANGED)
        """)
        subvertex.spec.write(data=self.n_input_dimensions)
        subvertex.spec.write(data=self.n_output_dimensions)
        subvertex.spec.write(data=subvertex.n_atoms)
        subvertex.spec.write(data=int(0.001 * 10**-6))
        subvertex.spec.write(data=self._tau_ref_in_steps)
        subvertex.spec.write(data=parameters.s1615(self._one_over_tau_rc))
        # subvertex.spec.write(data=... FILTER DECAY ...)
        # subvertex.spec.write(data=... FILTER DECAY COMPLEMENT ...)

    def write_region_bias(self, subvertex):
        """Write the bias region for the given subvertex."""
        subvertex.spec.switchWriteFocus(REGIONS.BIAS)
        for n in range(subvertex.lo_atom, subvertex.hi_atom + 1):
            # Write the bias for all atoms within this subvertex
            subvertex.spec.write(data=parameters.s1615(self.bias[n]))

    def write_region_encoders(self, subvertex):
        """Write the encoder region for the given subvertex."""
        subvertex.spec.switchWriteFocus(REGIONS.ENCODERS)
        subvertex.spec.comment("# Encoders Region")
        for n in range(subvertex.lo_atom, subvertex.hi_atom + 1):
            for d in range(self.data.n_input_dimensions):
                subvertex.spec.write(
                    data=parameters.s1615(
                        self.data.encoders[n, d] * self.data.gain[n]
                    )
                )

    def write_region_decoders(self, subvertex):
        """Write the decoder region for the given subvertex."""
        subvertex.spec.comment("# Decoders Region")
        subvertex.spec.switchWriteFocus(REGIONS.DECODERS)
        raise NotImplementedError

    def write_region_output_keys(self, subvertex):
        """Write the output keys region for the given subvertex."""
        subvertex.spec.comment("# Output Keys Region")
        subvertex.spec.switchWriteFocus(REGIONS.OUTPUT_KEYS)
        raise NotImplementedError
<|endoftext|>


<commit_before># Module:   dispatcher
# Date:     13th September 2007
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Dispatchers

This module implements URL dispatchers.
"""

import os

from circuits.core import Component

from cgifs import FieldStorage
from tools import expires, serve_file
from errors import HTTPError, NotFound
from utils import parseQueryString, dictform
from events import Request, Response

class DefaultDispatcher(Component):

    def __init__(self, docroot=None, defaults=None, **kwargs):
        super(DefaultDispatcher, self).__init__(**kwargs)

        if not docroot:
            self.docroot = os.getcwd()

        if not defaults:
            self.defaults = ["index.html"]

    def _getParams(self, request, response):
        body = request.body
        headers = request.headers

        if "Content-Type" not in headers:
            headers["Content-Type"] = ""

        try:
            form = FieldStorage(fp=body,
                headers=headers,
                environ={"REQUEST_METHOD": "POST"},
                keep_blank_values=True)
        except Exception, e:
            if e.__class__.__name__ == 'MaxSizeExceeded':
                # Post data is too big
                return HTTPError(request, response, 413)
            else:
                raise

        if form.file:
            return form.file
        else:
            return dictform(form)

    def _getChannel(self, path, method):
        """_getChannel(path, method) -> channel

        Find and return an appropriate channel path and method.

        The channel is found by traversing the system's event channels,
        and matching path components to successive channels in the system.

        If a channel cannot be found for a given path, but there is
        a default channel, then this will be used.

        The following is an example mappiny of path to channel given
        the following channels:

        Channels:
         /:index
         /foo:index
         /foo:hello
         /bar:GET
         /bar:POST
         /foo/bar/hello

        Method    Path                   Channel            VPath
        ---------------------------------------------------------
        GET       /                      /                  []
        GET       /1/2/3                 /                  [1, 2, 3]
        GET       /foo                   /foo               []
        GET       /foo/hello             /foo/hello         []
        GET       /foo/1/2/3             /foo               [1, 2, 3]
        GET       /foo/hello/1/2/3       /foo/hello         [1, 2, 3]
        GET       /bar                   /bar               []
        GET       /bar/1/2/3             /bar               [1, 2, 3]
        POST      /bar                   /bar               []
        POST      /bar/1/2/3             /bar               [1, 2, 3]
        GET       /foo/bar/hello         /foo/bar/hello     []
        GET       /foo/bar/hello/1/2/3   /foo/bar/hello     [1, 2, 3]
        """

        defaults = "index", method, "request"

        if not "/" in path:
            for default in defaults:
                k = "/:%s" % default
                if k in self.manager.channels:
                    return k, []
            return None, []

        names = [x for x in path.strip("/").split("/") if x]

        targets = set([x.split(":")[0] for x in self.manager.channels if x and \
                ":" in x and x[0] == "/"])

        i = 0
        matches = [""]
        candidates = []
        while i <= len(names):
            x = "/".join(matches) or "/"
            if x in targets:
                candidates.append([i, x])
                if i < len(names):
                    matches.append(names[i])
            else:
                break
            i += 1

        if not candidates:
            return None, []

        i, candidate = candidates.pop()

        if i < len(names):
            channels = [names[i], "index", method, "request"]
        else:
            channels = ["index", method, "request"]

        vpath = []
        channel = None
        for ch in channels:
            x = "%s:%s" % (candidate, ch)
            found = x in self.manager.channels
            if found:
                if i < len(names) and ch == names[i]:
                    i += 1
                channel = x
                break

        if channel is not None:
            if i < len(names):
                vpath = [x.replace("%2F", "/") for x in names[i:]]
            else:
                vpath = []

        return channel, vpath

    def request(self, request, response):
        path = request.path.strip("/")

        filename = None

        if path:
            filename = os.path.abspath(os.path.join(self.docroot, path))
        else:
            for default in self.defaults:
                filename = os.path.abspath(os.path.join(self.docroot, default))
                if os.path.exists(filename):
                    break

        if filename and os.path.exists(filename):
            expires(request, response, 3500*24*30)
            return serve_file(request, response, filename)

        channel, vpath = self._getChannel(request.path, request.method.upper())

        if channel:
            params = parseQueryString(request.qs)
            x = self._getParams(request, response)
            if not x:
                return
            elif type(x) == dict:
                params.update(x)
            else:
                request.body = x

            req = Request(request, response, *vpath, **params)

            try:
                responses = self.iter(req, channel)
                v = [x for x in responses if x is not None]
            except Exception, error:
                raise

            if v:
                return v[0]
            else:
                return NotFound(request, response)
        else:
            return NotFound(request, response)

class FileDispatcher(Component):

   template = """\

  <title>Index of %(path)s</title>

Index of %(path)s

%(files)s

"""

   def __init__(self, path=None, **kwargs):
      super(FileDispatcher, self).__init__(**kwargs)

      if not path:
         self.path = os.getcwd()

   def request(self, request, response):
      print request
      path = request.path.strip("/")
      print path

      if args:
         path = os.path.join("/", *args)
         real = os.path.join(self.path, *args)
      else:
         path = ""
         real = self.path

      if isfile(real):
         response.body = open(real, "r")
         return self.send(Response(request, response), "response", self.channel)

      data = {}
      data["path"] = path or "/"

      files = []

      if path:
         href = os.path.join(self.channel, path.lstrip("/"), "..")
         files.append("
.." % href)

      for file in listdir(real):
         href = os.path.join(self.channel, path.lstrip("/"), file)
         name = ("%s/" % file if isdir(file) else file)
         files.append("
%s" % (
            href, name))

      data["files"] = "
\n%s\n
" % "\n".join(files)

      response.headers["Content-Type"] = "text/html"
      return self.template % data

Dispatcher = DefaultDispatcher
FileServer = FileDispatcher
<commit_msg>lib.web.dispatchers: Fixed expiry.<commit_after># Module:   dispatcher
# Date:     13th September 2007
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Dispatchers

This module implements URL dispatchers.
"""

import os

from circuits.core import Component

from cgifs import FieldStorage
from tools import expires, serve_file
from errors import HTTPError, NotFound
from utils import parseQueryString, dictform
from events import Request, Response

class DefaultDispatcher(Component):

    def __init__(self, docroot=None, defaults=None, **kwargs):
        super(DefaultDispatcher, self).__init__(**kwargs)

        if not docroot:
            self.docroot = os.getcwd()

        if not defaults:
            self.defaults = ["index.html"]

    def _getParams(self, request, response):
        body = request.body
        headers = request.headers

        if "Content-Type" not in headers:
            headers["Content-Type"] = ""

        try:
            form = FieldStorage(fp=body,
                headers=headers,
                environ={"REQUEST_METHOD": "POST"},
                keep_blank_values=True)
        except Exception, e:
            if e.__class__.__name__ == 'MaxSizeExceeded':
                # Post data is too big
                return HTTPError(request, response, 413)
            else:
                raise

        if form.file:
            return form.file
        else:
            return dictform(form)

    def _getChannel(self, path, method):
        """_getChannel(path, method) -> channel

        Find and return an appropriate channel path and method.

        The channel is found by traversing the system's event channels,
        and matching path components to successive channels in the system.

        If a channel cannot be found for a given path, but there is
        a default channel, then this will be used.

        The following is an example mappiny of path to channel given
        the following channels:

        Channels:
         /:index
         /foo:index
         /foo:hello
         /bar:GET
         /bar:POST
         /foo/bar/hello

        Method    Path                   Channel            VPath
        ---------------------------------------------------------
        GET       /                      /                  []
        GET       /1/2/3                 /                  [1, 2, 3]
        GET       /foo                   /foo               []
        GET       /foo/hello             /foo/hello         []
        GET       /foo/1/2/3             /foo               [1, 2, 3]
        GET       /foo/hello/1/2/3       /foo/hello         [1, 2, 3]
        GET       /bar                   /bar               []
        GET       /bar/1/2/3             /bar               [1, 2, 3]
        POST      /bar                   /bar               []
        POST      /bar/1/2/3             /bar               [1, 2, 3]
        GET       /foo/bar/hello         /foo/bar/hello     []
        GET       /foo/bar/hello/1/2/3   /foo/bar/hello     [1, 2, 3]
        """

        defaults = "index", method, "request"

        if not "/" in path:
            for default in defaults:
                k = "/:%s" % default
                if k in self.manager.channels:
                    return k, []
            return None, []

        names = [x for x in path.strip("/").split("/") if x]

        targets = set([x.split(":")[0] for x in self.manager.channels if x and \
                ":" in x and x[0] == "/"])

        i = 0
        matches = [""]
        candidates = []
        while i <= len(names):
            x = "/".join(matches) or "/"
            if x in targets:
                candidates.append([i, x])
                if i < len(names):
                    matches.append(names[i])
            else:
                break
            i += 1

        if not candidates:
            return None, []

        i, candidate = candidates.pop()

        if i < len(names):
            channels = [names[i], "index", method, "request"]
        else:
            channels = ["index", method, "request"]

        vpath = []
        channel = None
        for ch in channels:
            x = "%s:%s" % (candidate, ch)
            found = x in self.manager.channels
            if found:
                if i < len(names) and ch == names[i]:
                    i += 1
                channel = x
                break

        if channel is not None:
            if i < len(names):
                vpath = [x.replace("%2F", "/") for x in names[i:]]
            else:
                vpath = []

        return channel, vpath

    def request(self, request, response):
        path = request.path.strip("/")

        filename = None

        if path:
            filename = os.path.abspath(os.path.join(self.docroot, path))
        else:
            for default in self.defaults:
                filename = os.path.abspath(os.path.join(self.docroot, default))
                if os.path.exists(filename):
                    break

        if filename and os.path.exists(filename):
            expires(request, response, 3600*24*30)
            return serve_file(request, response, filename)

        channel, vpath = self._getChannel(request.path, request.method.upper())

        if channel:
            params = parseQueryString(request.qs)
            x = self._getParams(request, response)
            if not x:
                return
            elif type(x) == dict:
                params.update(x)
            else:
                request.body = x

            req = Request(request, response, *vpath, **params)

            try:
                responses = self.iter(req, channel)
                v = [x for x in responses if x is not None]
            except Exception, error:
                raise

            if v:
                return v[0]
            else:
                return NotFound(request, response)
        else:
            return NotFound(request, response)

class FileDispatcher(Component):

   template = """\

  <title>Index of %(path)s</title>

Index of %(path)s

%(files)s

"""

   def __init__(self, path=None, **kwargs):
      super(FileDispatcher, self).__init__(**kwargs)

      if not path:
         self.path = os.getcwd()

   def request(self, request, response):
      print request
      path = request.path.strip("/")
      print path

      if args:
         path = os.path.join("/", *args)
         real = os.path.join(self.path, *args)
      else:
         path = ""
         real = self.path

      if isfile(real):
         response.body = open(real, "r")
         return self.send(Response(request, response), "response", self.channel)

      data = {}
      data["path"] = path or "/"

      files = []

      if path:
         href = os.path.join(self.channel, path.lstrip("/"), "..")
         files.append("
.." % href)

      for file in listdir(real):
         href = os.path.join(self.channel, path.lstrip("/"), file)
         name = ("%s/" % file if isdir(file) else file)
         files.append("
%s" % (
            href, name))

      data["files"] = "
\n%s\n
" % "\n".join(files)

      response.headers["Content-Type"] = "text/html"
      return self.template % data

Dispatcher = DefaultDispatcher
FileServer = FileDispatcher
<|endoftext|>


<commit_before>"""Use numpy testing framework which is based on nose as of v1.2

Extend the decorators to use nipy's gui and data labels.

"""

from numpy.testing.decorators import *

def make_label_dec(label, ds=None):
   """Factory function to create a decorator that applies one or more labels.

   :Parameters:
     label : string or sequence
     One or more labels that will be applied by the decorator to the functions
   it decorates.  Labels are attributes of the decorated function with their
   value set to True.

   :Keywords:
     ds : string
     An optional docstring for the resulting decorator.  If not given, a
     default docstring is auto-generated.

   :Returns:
     A decorator.

   :Examples:

   >>> from neuroimaging.testing import make_label_dec
   >>> slow = make_label_dec('slow')
   >>> print slow.__doc__
   Labels a test as 'slow'

   >>> from neuroimaging.testing import make_label_dec
   >>> rare = make_label_dec(['slow','hard'],
   ... "Mix labels 'slow' and 'hard' for rare tests")
   >>> @rare
   ... def f(): pass
   ...
   >>>
   >>> f.slow
   True
   >>> f.hard
   True
   """

   if isinstance(label,basestring):
       labels = [label]
   else:
       labels = label

   # Validate that the given label(s) are OK for use in setattr() by doing a
   # dry run on a dummy function.
   tmp = lambda : None
   for label in labels:
       setattr(tmp,label,True)

   # This is the actual decorator we'll return
   def decor(f):
       for label in labels:
           setattr(f,label,True)
       return f
   # Apply the user's docstring
   if ds is None:
       ds = "Labels a test as %r" % label
   decor.__doc__ = ds

   return decor

# Nipy specific labels
gui = make_label_dec('gui')
data = make_label_dec('data')

# We have a lot of known failures, sadly
def knownfailure(f):
   return knownfailureif(True)(f)

<commit_msg>ENH: Add decorator for skipping tests that need further review.<commit_after>"""Use numpy testing framework which is based on nose as of v1.2

Extend the decorators to use nipy's gui and data labels.

"""

from numpy.testing.decorators import *

def make_label_dec(label, ds=None):
   """Factory function to create a decorator that applies one or more labels.

   :Parameters:
     label : string or sequence
     One or more labels that will be applied by the decorator to the functions
   it decorates.  Labels are attributes of the decorated function with their
   value set to True.

   :Keywords:
     ds : string
     An optional docstring for the resulting decorator.  If not given, a
     default docstring is auto-generated.

   :Returns:
     A decorator.

   :Examples:

   >>> from neuroimaging.testing import make_label_dec
   >>> slow = make_label_dec('slow')
   >>> print slow.__doc__
   Labels a test as 'slow'

   >>> from neuroimaging.testing import make_label_dec
   >>> rare = make_label_dec(['slow','hard'],
   ... "Mix labels 'slow' and 'hard' for rare tests")
   >>> @rare
   ... def f(): pass
   ...
   >>>
   >>> f.slow
   True
   >>> f.hard
   True
   """

   if isinstance(label,basestring):
       labels = [label]
   else:
       labels = label

   # Validate that the given label(s) are OK for use in setattr() by doing a
   # dry run on a dummy function.
   tmp = lambda : None
   for label in labels:
       setattr(tmp,label,True)

   # This is the actual decorator we'll return
   def decor(f):
       for label in labels:
           setattr(f,label,True)
       return f
   # Apply the user's docstring
   if ds is None:
       ds = "Labels a test as %r" % label
   decor.__doc__ = ds

   return decor

# Nipy specific labels
gui = make_label_dec('gui')
data = make_label_dec('data')

# For tests that need further review
def needs_review(msg):
   """Skip a test that needs further review.

   Parameters
   ----------
   msg : string
       msg regarding the review that needs to be done

   """

   def skip_func(func):
      return skipif(True, msg)(func)
   return skip_func

# Easier version of the numpy knowfailure
def knownfailure(f):
   return knownfailureif(True)(f)

<|endoftext|>


<commit_before>from __future__ import absolute_import

# flake8: noqa

try:
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath                         # type: ignore

try:
    from contextlib import suppress
except ImportError:
    from contextlib2 import suppress                         # type: ignore

try:
    from functools import singledispatch
except ImportError:
    from singledispatch import singledispatch                   # type: ignore

try:
    from abc import ABC                                         # type: ignore
except ImportError:
    from abc import ABCMeta

    class ABC(object):                                          # type: ignore
        __metaclass__ = ABCMeta

try:
    FileNotFoundError = FileNotFoundError                       # type: ignore
except NameError:
    FileNotFoundError = OSError                                 # type: ignore

try:
    from importlib import metadata
except ImportError:
    import importlib_metadata as metadata  # type: ignore

try:
    from zipfile import Path as ZipPath  # type: ignore
except ImportError:
    from zipp import Path as ZipPath

class PackageSpec(object):
	def __init__(self, **kwargs):
		vars(self).update(kwargs)

def package_spec(package):
	return getattr(package, '__spec__', None) or \
		PackageSpec(
			origin=package.__file__,
			loader=getattr(package, '__loader__', None),
		)
<commit_msg>Need to ignore ZipPath on both branches because qa might be run on Python 3.7 or 3.8<commit_after>from __future__ import absolute_import

# flake8: noqa

try:
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath                         # type: ignore

try:
    from contextlib import suppress
except ImportError:
    from contextlib2 import suppress                         # type: ignore

try:
    from functools import singledispatch
except ImportError:
    from singledispatch import singledispatch                   # type: ignore

try:
    from abc import ABC                                         # type: ignore
except ImportError:
    from abc import ABCMeta

    class ABC(object):                                          # type: ignore
        __metaclass__ = ABCMeta

try:
    FileNotFoundError = FileNotFoundError                       # type: ignore
except NameError:
    FileNotFoundError = OSError                                 # type: ignore

try:
    from importlib import metadata
except ImportError:
    import importlib_metadata as metadata  # type: ignore

try:
    from zipfile import Path as ZipPath  # type: ignore
except ImportError:
    from zipp import Path as ZipPath  # type: ignore

class PackageSpec(object):
	def __init__(self, **kwargs):
		vars(self).update(kwargs)

def package_spec(package):
	return getattr(package, '__spec__', None) or \
		PackageSpec(
			origin=package.__file__,
			loader=getattr(package, '__loader__', None),
		)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import requests
import json
import sys
import getopt
import sqlite3
from board import Board
from configparser import ConfigParser
from urllib.parse import urlparse

application_key = ""
auth_token = ""
gtd_board_id = ""

db_file = 'trellonextactions.db'

def print_error_and_exit(message):
    sys.stderr.write(message + "\n")
    sys.exit(1)

def setup_database():
    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    c.execute('CREATE TABLE IF NOT EXISTS next_action ('
              'id INTEGER NOT NULL PRIMARY KEY, '
              'project_board_id VARCHAR(255) NOT NULL, '
              'project_next_action_id VARCHAR(255) NOT NULL, '
              'gtd_next_action_id VARCHAR(255) NOT NULL)')
    c.execute('CREATE INDEX IF NOT EXISTS project_board_id '
              'ON next_action (project_board_id)')
    c.execute('CREATE INDEX IF NOT EXISTS project_next_action_id '
              'ON next_action (project_next_action_id)')
    c.execute('CREATE INDEX IF NOT EXISTS gtd_next_action_id '
              'ON next_action (gtd_next_action_id)')

    conn.commit()
    conn.close()

def trello_create_card(name, description):
    try:
        list_id = get_list_id(gtd_board_id, 'Next Actions')
    except ValueError:
        print_error_and_exit("Could not find ID for Next Actions list")

    data = {
        'name': name,
        'desc': description + "\r\n\r\nAuto-created by TrelloNextActions",
        'idList': list_id,
        'key': application_key,
        'token': auth_token
    }

    response = trello_post_request('https://api.trello.com/1/cards/', data)
    return response['id']

def trello_delete_card(card_id):
    # Note: Only archive cards, just in case they've been edited by the user
    data = {
        'value': 'true',
        'key': application_key,
        'token': auth_token
    }

    response = trello_put_request('https://api.trello.com/1/cards/'
                                  + card_id + "/closed",
                                  data)

def sync_board(board):
    message_list = []

    # Figure out what cards already exist and so need no action
    exclude_list = []

    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    for next_action_card in board.nextActionList:
        c.execute('SELECT id '
                  'FROM next_action WHERE project_board_id = ? '
                  'AND project_next_action_id = ?',
                  (board.id, next_action_card['id']))
        next_action = c.fetchone()

        if next_action is not None:
            exclude_list.append(next_action_card['id'])

    # Delete everything else for this project board
    to_delete_query = ('SELECT gtd_next_action_id FROM next_action '
                       'WHERE project_board_id = ? ')
    parameter_list = [board.id]
    if len(exclude_list):
        placeholder_string = ','.join('?' * len(exclude_list))
        to_delete_query += ('AND project_next_action_id NOT IN ('
                            + placeholder_string + ')')
        parameter_list += exclude_list

    c.execute(to_delete_query, tuple(parameter_list))
    to_delete_list = c.fetchall()

    for to_delete_row in to_delete_list:
        trello_delete_card(to_delete_row[0])
        c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                  (to_delete_row[0],))
        message_list.append(board.name + ": Archived card " + to_delete_row[0])

    # Create any new cards
    for next_action_card in board.nextActionList:
        if next_action_card['id'] not in exclude_list:
            gtd_next_action_id = trello_create_card(board.name + " - "
                                                    + next_action_card['name'],
                                                    next_action_card['url'])
            c.execute('INSERT INTO next_action (project_board_id, '
                      'project_next_action_id, gtd_next_action_id) '
                      'VALUES (?, ?, ?)',
                      (board.id, next_action_card['id'], gtd_next_action_id))
            message_list.append(board.name + ": Created card "
                                + gtd_next_action_id)

    conn.commit()
    conn.close()

    return message_list

def reset():
    message_list = []

    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    c.execute('SELECT gtd_next_action_id FROM next_action')
    next_action_list = c.fetchall()
    for next_action in next_action_list:
        trello_delete_card(next_action[0])
        c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                  (next_action[0],))
        message_list.append("Archived card " + next_action[0])

    conn.commit()
    conn.close()

    return message_list

def trello_get_request(url):
    try:
        response = requests.get(url)
        return trello_handle_response(url, response)
    except:
        print_error_and_exit("Failed API request to " + url)

def trello_post_request(url, data):
    try:
        response = requests.post(url, data)
        return trello_handle_response(url, response)
    except:
        print_error_and_exit("Failed API request to " + url)

def trello_put_request(url, data):
    try:
        response = requests.put(url, data)
        return trello_handle_response(url, response)
    except:
        print_error_and_exit("Failed API request to " + url)

def trello_handle_response(url, response):
    if response.status_code != 200:
        print_error_and_exit("HTTP " + str(response.status_code)
                             + " response from " + url)

    try:
        json = response.json()
    except:
        print_error_and_exit("Could not parse JSON response from " + url)

    return json

def get_list_id(board_id, list_name):
    lists_on_board = trello_get_request('https://api.trello.com/1/boards/'
                                        + board_id + '/lists?cards=none'
                                        + '&key=' + application_key
                                        + '&token=' + auth_token)

    # Find the named list
    list_id = None
    for l in lists_on_board:
        if l['name'] == list_name:
            list_id = l['id']
            break

    if list_id is None:
        raise ValueError("No list with name '" + list_name + "' found")
    return list_id

def get_cards_in_list(board_id, list_name):
    list_id = get_list_id(board_id, list_name)

    # List the cards on that list
    cards_in_list = trello_get_request('https://api.trello.com/1/lists/'
                                       + list_id + '/cards'
                                       + '?key=' + application_key
                                       + '&token=' + auth_token)

    return cards_in_list

def get_next_action_per_project():
    next_action_list = []
    error_list = []

    try:
        project_card_list = get_cards_in_list(gtd_board_id, 'Projects')
    except ValueError as e:
        print_error_and_exit(str(e))

    for project_card in project_card_list:
        try:
            # The description of each card should contain the URL
            url = project_card['desc']

            # Parse the URL to get the short link
            url_bits = urlparse(url)
            # URLs all seem to be of the form /x/xyz123/nice-description
            path_bits = url_bits.path.split('/')
            short_id = path_bits[2]
            todo_card_list = get_cards_in_list(short_id, 'Todo')

            if len(todo_card_list) > 0:
                next_action_card = todo_card_list[0]
                next_action_list.append([project_card, next_action_card])
            else:
                error_list.append(project_card['name']
                                  + " - Todo list is empty")
        except ValueError as e:
            error_list.append(project_card['name'] + " - " + str(e))

    return next_action_list, error_list

def get_next_action_list():
    next_action_list = []
    error_list = []

    per_project_list, per_project_error_list = get_next_action_per_project()
    error_list += per_project_error_list

    for project_card, next_action_card in per_project_list:
        next_action_text = project_card['name'] + ' - '
        next_action_text += next_action_card['name']
        next_action_list.append(next_action_text)

    # Now return all the non-project Next Actions so that there's a
    # consolidated list
    next_action_card_list = get_cards_in_list(gtd_board_id, 'Next Actions')

    for next_action_card in next_action_card_list:
        next_action_list.append(next_action_card['name'])

    return next_action_list, error_list

def get_boards_with_owned_cards():
    board_map = {}

    card_list = trello_get_request('https://api.trello.com/1/members/me/cards/'
                                   + '?key=' + application_key
                                   + '&token=' + auth_token)
    for card in card_list:
        board_id = card['idBoard']
        if board_id not in board_map:
            board = trello_get_request('https://api.trello.com/1/boards/'
                                       + card['idBoard']
                                       + '?key=' + application_key
                                       + '&token=' + auth_token)
            board_map[board_id] = Board(board_id, board['name'])

        board_map[board_id].nextActionList.append(card)

    return board_map

def sync_next_actions():
    message_list = []
    error_list = []

    board_map = get_boards_with_owned_cards()

    per_project_list, per_project_error_list = get_next_action_per_project()
    error_list += per_project_error_list

    for project_card, next_action_card in per_project_list:
        board_id = next_action_card['idBoard']
        if board_id not in board_map:
            board_map[board_id] = Board(board_id, project_card['name'])

        board_map[board_id].nextActionList.append(next_action_card)

    for board in board_map:
        message_list += sync_board(board_map[board])

    # There may be some "orphaned" cards for boards that have no more next
    # actions /  owned cards
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('SELECT project_board_id, gtd_next_action_id '
              'FROM next_action')
    all_cards_list = c.fetchall()
    for card in all_cards_list:
        if card[0] not in board_map:
            trello_delete_card(card[1])
            c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                      (card[1],))
            message_list.append("Archived orphaned card " + card[1])

    conn.commit()
    conn.close()

    return message_list, error_list

def load_config(config_name):
    global application_key
    global auth_token
    global gtd_board_id

    config = ConfigParser()
    config.read(".trellonextactions")

    gtd_board_id = config.get(config_name, 'board_id')
    application_key = config.get(config_name, 'application_key')
    auth_token = config.get(config_name, 'auth_token')

def print_list(name, card_list):
    print(name + ":")
    print()
    for c in card_list:
        print(" [ ] " + c)
    print()

def main():
    config_name = 'default'
    action = 'list'

    try:
        opts, args = getopt.getopt(sys.argv[1:], "", ["config="])
        for opt, arg in opts:
            if opt == '--config':
                config_name = arg
    except getopt.GetoptError as e:
        print_error_and_exit(str(e))

    if len(args) > 1:
        print_error_and_exit("Too many arguments supplied")
    elif len(args) == 1:
        action = args[0]

    try:
        load_config(config_name)
    except:
        print_error_and_exit("Failed to load config '" + config_name + "'")

    try:
        setup_database()
    except Exception as e:
        print_error_and_exit("Error setting up DB: " + str(e))

    if action == 'list':
        next_action_list, error_list = get_next_action_list()

        print_list('Next Actions', next_action_list)
        if len(error_list) > 0:
            print_list('Errors', error_list)

    elif action == 'sync':
        message_list, error_list = sync_next_actions()

        if len(message_list) > 0:
            print_list('Messages', message_list)
        if len(error_list) > 0:
            print_list('Errors', error_list)

    elif action == 'reset':
        message_list = reset()
        print_list("Reset", message_list)

    else:
        print_error_and_exit("Unrecognised action '" + action + "'")

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fix bare except warnings<commit_after>#!/usr/bin/env python3
import requests
import json
import sys
import getopt
import sqlite3
from board import Board
from configparser import ConfigParser
from urllib.parse import urlparse

application_key = ""
auth_token = ""
gtd_board_id = ""

db_file = 'trellonextactions.db'

def print_error_and_exit(message):
    sys.stderr.write(message + "\n")
    sys.exit(1)

def setup_database():
    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    c.execute('CREATE TABLE IF NOT EXISTS next_action ('
              'id INTEGER NOT NULL PRIMARY KEY, '
              'project_board_id VARCHAR(255) NOT NULL, '
              'project_next_action_id VARCHAR(255) NOT NULL, '
              'gtd_next_action_id VARCHAR(255) NOT NULL)')
    c.execute('CREATE INDEX IF NOT EXISTS project_board_id '
              'ON next_action (project_board_id)')
    c.execute('CREATE INDEX IF NOT EXISTS project_next_action_id '
              'ON next_action (project_next_action_id)')
    c.execute('CREATE INDEX IF NOT EXISTS gtd_next_action_id '
              'ON next_action (gtd_next_action_id)')

    conn.commit()
    conn.close()

def trello_create_card(name, description):
    try:
        list_id = get_list_id(gtd_board_id, 'Next Actions')
    except ValueError:
        print_error_and_exit("Could not find ID for Next Actions list")

    data = {
        'name': name,
        'desc': description + "\r\n\r\nAuto-created by TrelloNextActions",
        'idList': list_id,
        'key': application_key,
        'token': auth_token
    }

    response = trello_post_request('https://api.trello.com/1/cards/', data)
    return response['id']

def trello_delete_card(card_id):
    # Note: Only archive cards, just in case they've been edited by the user
    data = {
        'value': 'true',
        'key': application_key,
        'token': auth_token
    }

    response = trello_put_request('https://api.trello.com/1/cards/'
                                  + card_id + "/closed",
                                  data)

def sync_board(board):
    message_list = []

    # Figure out what cards already exist and so need no action
    exclude_list = []

    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    for next_action_card in board.nextActionList:
        c.execute('SELECT id '
                  'FROM next_action WHERE project_board_id = ? '
                  'AND project_next_action_id = ?',
                  (board.id, next_action_card['id']))
        next_action = c.fetchone()

        if next_action is not None:
            exclude_list.append(next_action_card['id'])

    # Delete everything else for this project board
    to_delete_query = ('SELECT gtd_next_action_id FROM next_action '
                       'WHERE project_board_id = ? ')
    parameter_list = [board.id]
    if len(exclude_list):
        placeholder_string = ','.join('?' * len(exclude_list))
        to_delete_query += ('AND project_next_action_id NOT IN ('
                            + placeholder_string + ')')
        parameter_list += exclude_list

    c.execute(to_delete_query, tuple(parameter_list))
    to_delete_list = c.fetchall()

    for to_delete_row in to_delete_list:
        trello_delete_card(to_delete_row[0])
        c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                  (to_delete_row[0],))
        message_list.append(board.name + ": Archived card " + to_delete_row[0])

    # Create any new cards
    for next_action_card in board.nextActionList:
        if next_action_card['id'] not in exclude_list:
            gtd_next_action_id = trello_create_card(board.name + " - "
                                                    + next_action_card['name'],
                                                    next_action_card['url'])
            c.execute('INSERT INTO next_action (project_board_id, '
                      'project_next_action_id, gtd_next_action_id) '
                      'VALUES (?, ?, ?)',
                      (board.id, next_action_card['id'], gtd_next_action_id))
            message_list.append(board.name + ": Created card "
                                + gtd_next_action_id)

    conn.commit()
    conn.close()

    return message_list

def reset():
    message_list = []

    conn = sqlite3.connect(db_file)
    c = conn.cursor()

    c.execute('SELECT gtd_next_action_id FROM next_action')
    next_action_list = c.fetchall()
    for next_action in next_action_list:
        trello_delete_card(next_action[0])
        c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                  (next_action[0],))
        message_list.append("Archived card " + next_action[0])

    conn.commit()
    conn.close()

    return message_list

def trello_get_request(url):
    try:
        response = requests.get(url)
        return trello_handle_response(url, response)
    except Exception:
        print_error_and_exit("Failed API request to " + url)

def trello_post_request(url, data):
    try:
        response = requests.post(url, data)
        return trello_handle_response(url, response)
    except Exception:
        print_error_and_exit("Failed API request to " + url)

def trello_put_request(url, data):
    try:
        response = requests.put(url, data)
        return trello_handle_response(url, response)
    except Exception:
        print_error_and_exit("Failed API request to " + url)

def trello_handle_response(url, response):
    if response.status_code != 200:
        print_error_and_exit("HTTP " + str(response.status_code)
                             + " response from " + url)

    try:
        json = response.json()
    except Exception:
        print_error_and_exit("Could not parse JSON response from " + url)

    return json

def get_list_id(board_id, list_name):
    lists_on_board = trello_get_request('https://api.trello.com/1/boards/'
                                        + board_id + '/lists?cards=none'
                                        + '&key=' + application_key
                                        + '&token=' + auth_token)

    # Find the named list
    list_id = None
    for l in lists_on_board:
        if l['name'] == list_name:
            list_id = l['id']
            break

    if list_id is None:
        raise ValueError("No list with name '" + list_name + "' found")
    return list_id

def get_cards_in_list(board_id, list_name):
    list_id = get_list_id(board_id, list_name)

    # List the cards on that list
    cards_in_list = trello_get_request('https://api.trello.com/1/lists/'
                                       + list_id + '/cards'
                                       + '?key=' + application_key
                                       + '&token=' + auth_token)

    return cards_in_list

def get_next_action_per_project():
    next_action_list = []
    error_list = []

    try:
        project_card_list = get_cards_in_list(gtd_board_id, 'Projects')
    except ValueError as e:
        print_error_and_exit(str(e))

    for project_card in project_card_list:
        try:
            # The description of each card should contain the URL
            url = project_card['desc']

            # Parse the URL to get the short link
            url_bits = urlparse(url)
            # URLs all seem to be of the form /x/xyz123/nice-description
            path_bits = url_bits.path.split('/')
            short_id = path_bits[2]
            todo_card_list = get_cards_in_list(short_id, 'Todo')

            if len(todo_card_list) > 0:
                next_action_card = todo_card_list[0]
                next_action_list.append([project_card, next_action_card])
            else:
                error_list.append(project_card['name']
                                  + " - Todo list is empty")
        except ValueError as e:
            error_list.append(project_card['name'] + " - " + str(e))

    return next_action_list, error_list

def get_next_action_list():
    next_action_list = []
    error_list = []

    per_project_list, per_project_error_list = get_next_action_per_project()
    error_list += per_project_error_list

    for project_card, next_action_card in per_project_list:
        next_action_text = project_card['name'] + ' - '
        next_action_text += next_action_card['name']
        next_action_list.append(next_action_text)

    # Now return all the non-project Next Actions so that there's a
    # consolidated list
    next_action_card_list = get_cards_in_list(gtd_board_id, 'Next Actions')

    for next_action_card in next_action_card_list:
        next_action_list.append(next_action_card['name'])

    return next_action_list, error_list

def get_boards_with_owned_cards():
    board_map = {}

    card_list = trello_get_request('https://api.trello.com/1/members/me/cards/'
                                   + '?key=' + application_key
                                   + '&token=' + auth_token)
    for card in card_list:
        board_id = card['idBoard']
        if board_id not in board_map:
            board = trello_get_request('https://api.trello.com/1/boards/'
                                       + card['idBoard']
                                       + '?key=' + application_key
                                       + '&token=' + auth_token)
            board_map[board_id] = Board(board_id, board['name'])

        board_map[board_id].nextActionList.append(card)

    return board_map

def sync_next_actions():
    message_list = []
    error_list = []

    board_map = get_boards_with_owned_cards()

    per_project_list, per_project_error_list = get_next_action_per_project()
    error_list += per_project_error_list

    for project_card, next_action_card in per_project_list:
        board_id = next_action_card['idBoard']
        if board_id not in board_map:
            board_map[board_id] = Board(board_id, project_card['name'])

        board_map[board_id].nextActionList.append(next_action_card)

    for board in board_map:
        message_list += sync_board(board_map[board])

    # There may be some "orphaned" cards for boards that have no more next
    # actions /  owned cards
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('SELECT project_board_id, gtd_next_action_id '
              'FROM next_action')
    all_cards_list = c.fetchall()
    for card in all_cards_list:
        if card[0] not in board_map:
            trello_delete_card(card[1])
            c.execute('DELETE FROM next_action WHERE gtd_next_action_id = ?',
                      (card[1],))
            message_list.append("Archived orphaned card " + card[1])

    conn.commit()
    conn.close()

    return message_list, error_list

def load_config(config_name):
    global application_key
    global auth_token
    global gtd_board_id

    config = ConfigParser()
    config.read(".trellonextactions")

    gtd_board_id = config.get(config_name, 'board_id')
    application_key = config.get(config_name, 'application_key')
    auth_token = config.get(config_name, 'auth_token')

def print_list(name, card_list):
    print(name + ":")
    print()
    for c in card_list:
        print(" [ ] " + c)
    print()

def main():
    config_name = 'default'
    action = 'list'

    try:
        opts, args = getopt.getopt(sys.argv[1:], "", ["config="])
        for opt, arg in opts:
            if opt == '--config':
                config_name = arg
    except getopt.GetoptError as e:
        print_error_and_exit(str(e))

    if len(args) > 1:
        print_error_and_exit("Too many arguments supplied")
    elif len(args) == 1:
        action = args[0]

    try:
        load_config(config_name)
    except Exception:
        print_error_and_exit("Failed to load config '" + config_name + "'")

    try:
        setup_database()
    except Exception as e:
        print_error_and_exit("Error setting up DB: " + str(e))

    if action == 'list':
        next_action_list, error_list = get_next_action_list()

        print_list('Next Actions', next_action_list)
        if len(error_list) > 0:
            print_list('Errors', error_list)

    elif action == 'sync':
        message_list, error_list = sync_next_actions()

        if len(message_list) > 0:
            print_list('Messages', message_list)
        if len(error_list) > 0:
            print_list('Errors', error_list)

    elif action == 'reset':
        message_list = reset()
        print_list("Reset", message_list)

    else:
        print_error_and_exit("Unrecognised action '" + action + "'")

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""This commandline module provides classes for interfacing with the
`ICA-AROMA.py<https://github.com/rhr-pruim/ICA-AROMA>`_ command line tool.
    Change directory to provide relative paths for doctests
    >>> import os
    >>> filepath = os.path.dirname(os.path.realpath(__file__))
    >>> datadir = os.path.realpath(os.path.join(filepath,
    ...                            '../../testing/data'))
    >>> os.chdir(datadir)
"""
from nipype.interfaces.base import (
    TraitedSpec,
    CommandLineInputSpec,
    CommandLine,
    File,
    Directory,
    traits,
    OutputMultiPath,
    isdefined
)
import os

class ICA_AROMAInputSpec(CommandLineInputSpec):
    feat_dir = Directory(exists=True, mandatory=True,
                         argstr='-feat %s',
                         xor=['in_file', 'mat_file', 'fnirt_warp_file', 'motion_parameters'],
                         desc='If a feat directory exists and temporal filtering '
                         'has not been run yet, ICA_AROMA can use the files in '
                         'this directory.')
    in_file = File(exists=True, mandatory=True,
                   argstr='-i %s', xor=['feat_dir'],
                   desc='volume to be denoised')
    out_dir = Directory('out', genfile=True,
                        argstr='-o %s',
                        desc='output directory')
    mask = File(exists=True, argstr='-m %s', xor=['feat_dir'],
                desc='path/name volume mask')
    dim = traits.Int(argstr='-dim %d',
                     desc='Dimensionality reduction when running '
                     'MELODIC (defualt is automatic estimation)')
    TR = traits.Float(argstr='-tr %.3f',
                      desc='TR in seconds. If this is not specified '
                      'the TR will be extracted from the '
                      'header of the fMRI nifti file.')
    melodic_dir = Directory(exists=True, argstr='-meldir %s',
                            desc='path to MELODIC directory if MELODIC has already been run')
    mat_file = File(exists=True, argstr='-affmat %s', xor=['feat_dir'],
                    desc='path/name of the mat-file describing the '
                    'affine registration (e.g. FSL FLIRT) of the '
                    'functional data to structural space (.mat file)')
    fnirt_warp_file = File(exists=True, argstr='-warp %s', xor=['feat_dir'],
                           desc='File name of the warp-file describing '
                           'the non-linear registration (e.g. FSL FNIRT) '
                           'of the structural data to MNI152 space (.nii.gz)')
    motion_parameters = File(exists=True, mandatory=True,
                             argstr='-mc %s', xor=['feat_dir'],
                             desc='motion parameters file')
    denoise_type = traits.Enum('nonaggr', 'aggr', 'both', 'no', usedefault=True,
                               mandatory=True, argstr='-den %s',
                               desc='Type of denoising strategy:\n'
                               '-none: only classification, no denoising\n'
                               '-nonaggr (default): non-aggresssive denoising, i.e. partial component regression\n'
                               '-aggr: aggressive denoising, i.e. full component regression\n'
                               '-both: both aggressive and non-aggressive denoising (two outputs)')

class ICA_AROMAOutputSpec(TraitedSpec):
    aggr_denoised_file = File(exists=True,
                              desc='if generated: aggressively denoised volume')
    nonaggr_denoised_file = File(exists=True,
                                 desc='if generated: non aggressively denoised volume' )
    out_dir = Directory(exists=True,
                        desc='directory contains (in addition to the denoised files): '
                        'melodic.ica + classified_motion_components + '
                        'classification_overview + feature_scores + melodic_ic_mni)')

class ICA_AROMA(CommandLine):
    """
    Interface for the ICA_AROMA.py script.

    ICA-AROMA (i.e. 'ICA-based Automatic Removal Of Motion Artifacts') concerns
    a data-driven method to identify and remove motion-related independent
    components from fMRI data. To that end it exploits a small, but robust
    set of theoretically motivated features, preventing the need for classifier
    re-training and therefore providing direct and easy applicability.

    See link for further documentation: https://github.com/rhr-pruim/ICA-AROMA

    Example
    -------

    >>> from nipype.interfaces.fsl import ICA_AROMA
    >>> from nipype.testing import example_data
    >>> AROMA_obj = ICA_AROMA.ICA_AROMA()
    >>> AROMA_obj.inputs.in_file = 'functional.nii'
    >>> AROMA_obj.inputs.mat_file = 'func_to_struct.mat'
    >>> AROMA_obj.inputs.fnirt_warp_file = 'warpfield.nii'
    >>> AROMA_obj.inputs.motion_parameters = 'fsl_mcflirt_movpar.txt'
    >>> AROMA_obj.inputs.mask = 'mask.nii.gz'
    >>> AROMA_obj.inputs.denoise_type = 'both'
    >>> AROMA_obj.inputs.out_dir = 'ICA_testout'
    >>> AROMA_obj.cmdline # doctest: +ALLOW_UNICODE
    'ICA_AROMA.py -den both -warp warpfield.nii -i functional.nii -m mask.nii.gz -affmat func_to_struct.mat -mc fsl_mcflirt_movpar.txt -o ICA_testout'
    """
    _cmd = 'ICA_AROMA.py'
    input_spec = ICA_AROMAInputSpec
    output_spec = ICA_AROMAOutputSpec

    def _list_outputs(self):
        outputs = self.output_spec().get()
        if isdefined(self.inputs.out_dir):
            outputs['out_dir'] = os.path.abspath(self.inputs.out_dir)
        else:
            outputs['out_dir'] = self._gen_filename("out_dir")
        out_dir = os.path.abspath(self.inputs.out_dir)
        outputs['out_dir'] = out_dir

        if self.inputs.denoise_type in ('aggr', 'both'):
            outputs['aggr_denoised_file'] = os.path.join(out_dir, 'denoised_func_data_aggr.nii.gz')
        if self.inputs.denoise_type in ('nonaggr', 'both'):
            outputs['nonaggr_denoised_file'] = os.path.join(out_dir, 'denoised_func_data_nonaggr.nii.gz')

    def _gen_filename(self, name):
        if name == "out_dir":
            return os.getcwd()

        return outputs
<commit_msg>ENH: ICA_AROMA default directory fix<commit_after># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""This commandline module provides classes for interfacing with the
`ICA-AROMA.py<https://github.com/rhr-pruim/ICA-AROMA>`_ command line tool.
    Change directory to provide relative paths for doctests
    >>> import os
    >>> filepath = os.path.dirname(os.path.realpath(__file__))
    >>> datadir = os.path.realpath(os.path.join(filepath,
    ...                            '../../testing/data'))
    >>> os.chdir(datadir)
"""
from nipype.interfaces.base import (
    TraitedSpec,
    CommandLineInputSpec,
    CommandLine,
    File,
    Directory,
    traits,
    OutputMultiPath,
    isdefined
)
import os

class ICA_AROMAInputSpec(CommandLineInputSpec):
    feat_dir = Directory(exists=True, mandatory=True,
                         argstr='-feat %s',
                         xor=['in_file', 'mat_file', 'fnirt_warp_file', 'motion_parameters'],
                         desc='If a feat directory exists and temporal filtering '
                         'has not been run yet, ICA_AROMA can use the files in '
                         'this directory.')
    in_file = File(exists=True, mandatory=True,
                   argstr='-i %s', xor=['feat_dir'],
                   desc='volume to be denoised')
    out_dir = Directory('out', genfile=True,
                        argstr='-o %s',
                        desc='output directory')
    mask = File(exists=True, argstr='-m %s', xor=['feat_dir'],
                desc='path/name volume mask')
    dim = traits.Int(argstr='-dim %d',
                     desc='Dimensionality reduction when running '
                     'MELODIC (defualt is automatic estimation)')
    TR = traits.Float(argstr='-tr %.3f',
                      desc='TR in seconds. If this is not specified '
                      'the TR will be extracted from the '
                      'header of the fMRI nifti file.')
    melodic_dir = Directory(exists=True, argstr='-meldir %s',
                            desc='path to MELODIC directory if MELODIC has already been run')
    mat_file = File(exists=True, argstr='-affmat %s', xor=['feat_dir'],
                    desc='path/name of the mat-file describing the '
                    'affine registration (e.g. FSL FLIRT) of the '
                    'functional data to structural space (.mat file)')
    fnirt_warp_file = File(exists=True, argstr='-warp %s', xor=['feat_dir'],
                           desc='File name of the warp-file describing '
                           'the non-linear registration (e.g. FSL FNIRT) '
                           'of the structural data to MNI152 space (.nii.gz)')
    motion_parameters = File(exists=True, mandatory=True,
                             argstr='-mc %s', xor=['feat_dir'],
                             desc='motion parameters file')
    denoise_type = traits.Enum('nonaggr', 'aggr', 'both', 'no', usedefault=True,
                               mandatory=True, argstr='-den %s',
                               desc='Type of denoising strategy:\n'
                               '-none: only classification, no denoising\n'
                               '-nonaggr (default): non-aggresssive denoising, i.e. partial component regression\n'
                               '-aggr: aggressive denoising, i.e. full component regression\n'
                               '-both: both aggressive and non-aggressive denoising (two outputs)')

class ICA_AROMAOutputSpec(TraitedSpec):
    aggr_denoised_file = File(exists=True,
                              desc='if generated: aggressively denoised volume')
    nonaggr_denoised_file = File(exists=True,
                                 desc='if generated: non aggressively denoised volume' )
    out_dir = Directory(exists=True,
                        desc='directory contains (in addition to the denoised files): '
                        'melodic.ica + classified_motion_components + '
                        'classification_overview + feature_scores + melodic_ic_mni)')

class ICA_AROMA(CommandLine):
    """
    Interface for the ICA_AROMA.py script.

    ICA-AROMA (i.e. 'ICA-based Automatic Removal Of Motion Artifacts') concerns
    a data-driven method to identify and remove motion-related independent
    components from fMRI data. To that end it exploits a small, but robust
    set of theoretically motivated features, preventing the need for classifier
    re-training and therefore providing direct and easy applicability.

    See link for further documentation: https://github.com/rhr-pruim/ICA-AROMA

    Example
    -------

    >>> from nipype.interfaces.fsl import ICA_AROMA
    >>> from nipype.testing import example_data
    >>> AROMA_obj = ICA_AROMA.ICA_AROMA()
    >>> AROMA_obj.inputs.in_file = 'functional.nii'
    >>> AROMA_obj.inputs.mat_file = 'func_to_struct.mat'
    >>> AROMA_obj.inputs.fnirt_warp_file = 'warpfield.nii'
    >>> AROMA_obj.inputs.motion_parameters = 'fsl_mcflirt_movpar.txt'
    >>> AROMA_obj.inputs.mask = 'mask.nii.gz'
    >>> AROMA_obj.inputs.denoise_type = 'both'
    >>> AROMA_obj.inputs.out_dir = 'ICA_testout'
    >>> AROMA_obj.cmdline # doctest: +ALLOW_UNICODE
    'ICA_AROMA.py -den both -warp warpfield.nii -i functional.nii -m mask.nii.gz -affmat func_to_struct.mat -mc fsl_mcflirt_movpar.txt -o ICA_testout'
    """
    _cmd = 'ICA_AROMA.py'
    input_spec = ICA_AROMAInputSpec
    output_spec = ICA_AROMAOutputSpec

    def _list_outputs(self):
        outputs = self.output_spec().get()
        if isdefined(self.inputs.out_dir):
            outputs['out_dir'] = os.path.abspath(self.inputs.out_dir)
        else:
            outputs['out_dir'] = self._gen_filename('out_dir')
        out_dir = outputs['out_dir']

        if self.inputs.denoise_type in ('aggr', 'both'):
            outputs['aggr_denoised_file'] = os.path.join(out_dir, 'denoised_func_data_aggr.nii.gz')
        if self.inputs.denoise_type in ('nonaggr', 'both'):
            outputs['nonaggr_denoised_file'] = os.path.join(out_dir, 'denoised_func_data_nonaggr.nii.gz')
        return outputs

    def _gen_filename(self, name):
        if name == 'out_dir':
            return os.getcwd()

<|endoftext|>


<commit_before>'''
Variant of node_wrapper that fetches and stores data on amazon's S3 using boto
'''

import os
import hashlib
from tempfile import mkdtemp
from ConfigParser import RawConfigParser

# Eventually, we'll want to hoist this stuff to something in nipype.utils
# But wait 'til it stabilizes here!
try:
    from boto.s3 import Connection
    import tables
    from boto.s3.key import Key
except:
    pass

class S3NodeWrapper(object):
    '''
    Wrapper for interface objects, done in a different style than NodeWrapper

    The wrapping is meant to achieve the following:
    1) Quarantine any processing such that original files are not modifiable by
    squirrely analysis programs
    2) Keep copies of things in such a way that it's easy to tell if something
    has already been done

    In particular, S3NodeWrapper introduces the notion of a URI to the
    identification of data resources
    Note that the logic here assumes that files in S3 do not change. This is
    somewhat redundant with the notion of URI.

    I'm also not implementing features until they are actually useful

    Parameters
    ----------
    interface : Interface instance
        e.g., fsl.Bet(frac=0.34), spm.Coregister()
    s3_storage : string
        root directory for s3 files
    base_directory : string
        root directory where working directories will be created
        two S3NodeWrappers using the same base_directory will share cached
        files already downloaded from s3.
        default=None, which results in the use of mkdtemp
    '''
    @property
    def inputs(self):
        return self.interface.inputs

    def __init__(self, interface, s3_directory, working_directory=None):
        self.interface = interface
        self.s3_directory = s3_directory
        self.base_directory = base_directory

# Candidate for hoisting
class S3Transporter(object):
    '''Connect one working directory with one URI root

    If you want more than one mapping, make more than one class

    Currently, only s3:// syntax is supported. It should be trivial to
    implement other formats, e.g. http, ftp, scp...

    There is also no error checking anywhere at this point. This should clearly
    be added at some point'''

    def __init__(self, working_directory, s3_root, s3env='shell'):
        '''Set up the environment for dealing with S3'''
        if s3env == 'shell':
            self.access_key = os.environ['AWS_ACCESS_KEY']
            self.secret_key = os.environ['AWS_SECRET_KEY']
        if s3env == 's3cfg':
            rcfg = RawConfigParser()
            rcfg.read(os.environ['HOME'] + '/.s3cfg')
            config = dict(rcfg.items('default'))
            self.access_key = config['access_key']
            self.secret_key = config['secret_key']
            # This stuff still needs to be done
            gpg_command = config['gpg_command']

        if self.access_key is None:
            raise Exception('AWS_ACCESS_KEY is not defined')
        if self.secret_key is None:
            raise Exception('AWS_SECRET_KEY is not defined')

        self.working_directory = working_directory
        try:
            self.s3_bucket, self.s3_key_prefix = \
                    s3_root.strip('/').split('/', 1)
        except ValueError:
            self.s3_bucket = s3_bucket.strip('/')
            self.s3_key_prefix = ''

        self.conn = Connection(aws_access_key_id=self.access_key,
                       aws_secret_access_key=self.secret_key)
        self.bucket = self.conn.get_bucket(self.s3_bucket)

    def get(self, s3_path, local_path=None):
        '''Get a file corresponding to uri

        Currently, authentication is handled by the standard amazon S3 environment
        variables for public and private keys.

        Parameters
        ----------
        s3_path : string
            path relative to the s3_root specified at __init__ time
        local_path : string
            Where to put the downloaded file
        '''
        if local_path is None:
            local_path = s3_path
        full_key = os.path.join(self.s3_key_prefix, s3_path)
        key = self.bucket.get_key(full_key)
        full_fname = os.path.join(self.working_directory, local_path)
        self.make_dirs(os.path.dirname(full_fname))
        f = open(full_fname, 'w')
        key.get_file(f)
        f.close()

    def put(self, s3_path, local_path):
        '''Put file from local path to s3 path'''
        if s3_path is None:
            s3_path = ''
        full_key = os.path.join(self.s3_key_prefix, s3_path)
        key = Key(self.bucket)
        key.key = full_key
        f = open(local_path, 'r')
        key.send_file(f)
        f.close()

    def make_dirs(self, path):
        '''Create intervening subdirectories as needed'''
        if not os.path.exists(path):
            self.make_dirs(os.path.dirname(path))
            os.mkdir(path)

<commit_msg>Catch ImportError's only in import-try block.<commit_after>'''
Variant of node_wrapper that fetches and stores data on amazon's S3 using boto
'''

import os
import hashlib
from tempfile import mkdtemp
from ConfigParser import RawConfigParser

# Eventually, we'll want to hoist this stuff to something in nipype.utils
# But wait 'til it stabilizes here!
try:
    from boto.s3 import Connection
    import tables
    from boto.s3.key import Key
except ImportError:
    pass

class S3NodeWrapper(object):
    '''
    Wrapper for interface objects, done in a different style than NodeWrapper

    The wrapping is meant to achieve the following:
    1) Quarantine any processing such that original files are not modifiable by
    squirrely analysis programs
    2) Keep copies of things in such a way that it's easy to tell if something
    has already been done

    In particular, S3NodeWrapper introduces the notion of a URI to the
    identification of data resources
    Note that the logic here assumes that files in S3 do not change. This is
    somewhat redundant with the notion of URI.

    I'm also not implementing features until they are actually useful

    Parameters
    ----------
    interface : Interface instance
        e.g., fsl.Bet(frac=0.34), spm.Coregister()
    s3_storage : string
        root directory for s3 files
    base_directory : string
        root directory where working directories will be created
        two S3NodeWrappers using the same base_directory will share cached
        files already downloaded from s3.
        default=None, which results in the use of mkdtemp
    '''
    @property
    def inputs(self):
        return self.interface.inputs

    def __init__(self, interface, s3_directory, working_directory=None):
        self.interface = interface
        self.s3_directory = s3_directory
        self.base_directory = base_directory

# Candidate for hoisting
class S3Transporter(object):
    '''Connect one working directory with one URI root

    If you want more than one mapping, make more than one class

    Currently, only s3:// syntax is supported. It should be trivial to
    implement other formats, e.g. http, ftp, scp...

    There is also no error checking anywhere at this point. This should clearly
    be added at some point'''

    def __init__(self, working_directory, s3_root, s3env='shell'):
        '''Set up the environment for dealing with S3'''
        if s3env == 'shell':
            self.access_key = os.environ['AWS_ACCESS_KEY']
            self.secret_key = os.environ['AWS_SECRET_KEY']
        if s3env == 's3cfg':
            rcfg = RawConfigParser()
            rcfg.read(os.environ['HOME'] + '/.s3cfg')
            config = dict(rcfg.items('default'))
            self.access_key = config['access_key']
            self.secret_key = config['secret_key']
            # This stuff still needs to be done
            gpg_command = config['gpg_command']

        if self.access_key is None:
            raise Exception('AWS_ACCESS_KEY is not defined')
        if self.secret_key is None:
            raise Exception('AWS_SECRET_KEY is not defined')

        self.working_directory = working_directory
        try:
            self.s3_bucket, self.s3_key_prefix = \
                    s3_root.strip('/').split('/', 1)
        except ValueError:
            self.s3_bucket = s3_bucket.strip('/')
            self.s3_key_prefix = ''

        self.conn = Connection(aws_access_key_id=self.access_key,
                       aws_secret_access_key=self.secret_key)
        self.bucket = self.conn.get_bucket(self.s3_bucket)

    def get(self, s3_path, local_path=None):
        '''Get a file corresponding to uri

        Currently, authentication is handled by the standard amazon S3 environment
        variables for public and private keys.

        Parameters
        ----------
        s3_path : string
            path relative to the s3_root specified at __init__ time
        local_path : string
            Where to put the downloaded file
        '''
        if local_path is None:
            local_path = s3_path
        full_key = os.path.join(self.s3_key_prefix, s3_path)
        key = self.bucket.get_key(full_key)
        full_fname = os.path.join(self.working_directory, local_path)
        self.make_dirs(os.path.dirname(full_fname))
        f = open(full_fname, 'w')
        key.get_file(f)
        f.close()

    def put(self, s3_path, local_path):
        '''Put file from local path to s3 path'''
        if s3_path is None:
            s3_path = ''
        full_key = os.path.join(self.s3_key_prefix, s3_path)
        key = Key(self.bucket)
        key.key = full_key
        f = open(local_path, 'r')
        key.send_file(f)
        f.close()

    def make_dirs(self, path):
        '''Create intervening subdirectories as needed'''
        if not os.path.exists(path):
            self.make_dirs(os.path.dirname(path))
            os.mkdir(path)

<|endoftext|>


<commit_before>from django.forms.widgets import Widget, TextInput
from django.forms.utils import flatatt
from django.utils.html import format_html
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from taggit.utils import edit_string_for_tags

from nodeconductor.openstack import Types
from nodeconductor.openstack.models import Instance

class OpenStackTagsWidget(Widget):

    def render(self, name, value, attrs=None):
        self.tag_name = name
        self.tag_value = value or []

        rows = '
'.join([
            self.render_license('OS', Types.PriceItems.LICENSE_OS, Types.Os.CHOICES),
            self.render_license('Application', Types.PriceItems.LICENSE_APPLICATION, Types.Applications.CHOICES),
            self.render_license('Support', Types.PriceItems.SUPPORT, Types.Support.CHOICES),
            '',  # extra line
            self.render_remote(),
            '
Extra tags: %s' % TextInput().render(
                name, edit_string_for_tags([t.tag for t in self.tag_value]), attrs={'size': 50})
        ])

        return mark_safe('
%s
' % rows)

    def render_remote(self):
        tags = []
        type_value = instance_value = ''
        for tag in self.tag_value:
            if tag.tag.name.startswith('remote-'):
                type_value, instance_value = tag.tag.name.split(':')
            else:
                tags.append(tag)

        self.tag_value = tags

        choices = ('remote-origin', 'Origin'), ('remote-copy', 'Copy')
        field_name = '%s_%s' % (self.tag_name, 'remote_type')
        final_attrs = self.build_attrs(None, name=field_name)
        output = [format_html('Remote: <select{}>', flatatt(final_attrs))]
        output.append('<option value="">None</option>')
        for opt, txt in choices:
            selected = ' selected="selected"' if opt == type_value else ''
            output.append(format_html(
                '<option value="{}"{}>{}</option>', opt, selected, force_text(txt)))
        output.append('</select>')

        field_name = '%s_%s' % (self.tag_name, 'remote_instance')
        final_attrs = self.build_attrs(None, name=field_name, value=instance_value, placeholder="Instance UUID")
        html = '\n'.join(output) + format_html('<input{} />', flatatt(final_attrs))

        return html

    def render_license(self, title, name, choices):
        field_name = '%s_%s' % (self.tag_name, title.lower())
        final_attrs = self.build_attrs(None, type='hidden', name=field_name, value=name)
        html = format_html('{}: <input{} />', title, flatatt(final_attrs))

        tags = []
        val1 = val2 = ''
        for tag in self.tag_value:
            if tag.tag.name.startswith(name):
                opts = tag.tag.name.split(':')[1:]
                if len(opts) > 1:
                    val1, val2 = opts
                else:
                    val1 = opts[0]
                    val2 = dict(choices).get(val1)
            else:
                tags.append(tag)

        self.tag_value = tags

        final_attrs = self.build_attrs(None, name=field_name)
        output = [format_html('<select{}>', flatatt(final_attrs))]
        output.append('<option value="">None</option>')
        for opt, txt in choices:
            selected = ' selected="selected"' if opt == val1 else ''
            output.append(format_html(
                '<option value="{}"{}>{}</option>', opt, selected, force_text(txt)))
        output.append('</select>')

        html += '\n'.join(output)

        final_attrs = self.build_attrs(None, type='text', name=field_name, value=val2, placeholder='Custom name')
        html += format_html('<input{} />', flatatt(final_attrs))

        return html
<commit_msg>Finetune<commit_after>from django.forms.widgets import Widget, TextInput
from django.forms.utils import flatatt
from django.utils.html import format_html
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from taggit.utils import edit_string_for_tags

from nodeconductor.openstack import Types
from nodeconductor.openstack.models import Instance

class OpenStackTagsWidget(Widget):

    def render(self, name, value, attrs=None):
        self.tag_name = name
        self.tag_value = value or []

        rows = '
'.join([
            self.render_license('OS', Types.PriceItems.LICENSE_OS, Types.Os.CHOICES),
            self.render_license('Application', Types.PriceItems.LICENSE_APPLICATION, Types.Applications.CHOICES),
            self.render_license('Support', Types.PriceItems.SUPPORT, Types.Support.CHOICES),
            '',  # extra line
            self.render_remote(),
            '
Extra tags: %s' % TextInput().render(
                name, edit_string_for_tags([t.tag for t in self.tag_value]), attrs={'size': 50})
        ])

        return mark_safe('
%s
' % rows)

    def render_remote(self):
        tags = []
        type_value = instance_value = ''
        for tag in self.tag_value:
            if tag.tag.name.startswith('remote-'):
                type_value, instance_value = tag.tag.name.split(':')
            else:
                tags.append(tag)

        self.tag_value = tags

        choices = ('remote-origin', 'Origin'), ('remote-copy', 'Copy')
        field_name = '%s_%s' % (self.tag_name, 'remote_type')
        final_attrs = self.build_attrs(None, name=field_name)
        output = [format_html('Remote: <select{}>', flatatt(final_attrs))]
        output.append('<option value="">None</option>')
        for opt, txt in choices:
            selected = ' selected="selected"' if opt == type_value else ''
            output.append(format_html(
                '<option value="{}"{}>{}</option>', opt, selected, force_text(txt)))
        output.append('</select>')

        field_name = '%s_%s' % (self.tag_name, 'remote_instance')
        final_attrs = self.build_attrs(None, name=field_name, size=32,
                                       value=instance_value, placeholder="Instance UUID")
        html = '\n'.join(output) + format_html('<input{} />', flatatt(final_attrs))

        return html

    def render_license(self, title, name, choices):
        field_name = '%s_%s' % (self.tag_name, title.lower())
        final_attrs = self.build_attrs(None, type='hidden', name=field_name, value=name)
        html = format_html('{}: <input{} />', title, flatatt(final_attrs))

        tags = []
        val1 = val2 = ''
        for tag in self.tag_value:
            if tag.tag.name.startswith(name):
                opts = tag.tag.name.split(':')[1:]
                if len(opts) > 1:
                    val1, val2 = opts
                else:
                    val1 = opts[0]
                    val2 = dict(choices).get(val1)
            else:
                tags.append(tag)

        self.tag_value = tags

        final_attrs = self.build_attrs(None, name=field_name)
        output = [format_html('<select{}>', flatatt(final_attrs))]
        output.append('<option value="">None</option>')
        for opt, txt in choices:
            selected = ' selected="selected"' if opt == val1 else ''
            output.append(format_html(
                '<option value="{}"{}>{}</option>', opt, selected, force_text(txt)))
        output.append('</select>')

        html += '\n'.join(output)

        final_attrs = self.build_attrs(None, type='text', name=field_name, value=val2, placeholder='Custom name')
        html += format_html('<input{} />', flatatt(final_attrs))

        return html
<|endoftext|>