<commit_before># vim: set et ts=4 sw=4 fdm=marker
"""
MIT License

Copyright (c) 2016 Jesse Hogan

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from pdb import set_trace; B=set_trace
from random import randint

class entities(object):
    def __init__(self):
        self.clear()

    def clear(self):
        self._ls=[]

    def __iter__(self):
        for t in self._list:
            yield t

    def getrandom(self):
        if self.isempty: return None
        ix = randint(0, self.ubound)
        self[ix]

    def where(self, fn):
        es = entities()
        for e in self:
            if fn(e): es += e
        return es

    def remove(self, e):
        if callable(e):
            rms = self.where(e)
        else:
            rms = [e]

        for i, e1 in enumerate(self):
            for e2 in rms:
                if e1 is e2:
                    del self._list[i]
                    break

    def reversed(self):
        r = type(self)()
        for e in reversed(self._ls):
            r += e
        return r

    @property
    def ubound(self):
        if self.isempty: return None
        return self.count - 1

    def shift(self):
        return self._ls.pop(0)

    def unshift(self, t):
        # TODO: Return entities object to indicate what was unshifted
        return self._ls.insert(0, t)

    def __lshift__(self, a):
        self.unshift(a)

    def append(self, obj, uniq=False, r=None):
        if not r: r = []
        if isinstance(obj, entity):
            t = obj
        elif isinstance(obj, entities):
            for t in obj:
                if uniq:
                    for t1 in self:
                        if t is t1: break
                    else:
                        self.append(t, r=r)
                        continue
                    break
                else:
                    self.append(t, r=r)
            return r
        else:
            raise ValueError('Unsupported object appended')

        if uniq:
            for t1 in self:
                if t is t1: return r

        r.append(t)
        for t in r:
            self._list.append(t)

        return r

    def __iadd__(self, t):
        self.append(t)
        return self

    def __iand__(self, t):
        self.append(t, uniq=True)
        return self

    def add(self, ts):
        self += ts
        return self

    def __add__(self, t):
        return self.add(t)

    @property
    def _list(self):
        if not hasattr(self, '_ls'):
            self._ls = []
        return self._ls

    @property
    def count(self):
        return len(self._list)

    def __len__(self):
        return self.count

    @property
    def isempty(self):
        return self.count == 0

    @property
    def ispopulated(self):
        return not self.isempty

    def __str__(self):
        if not self.isempty:
            r=''
            for i, t in enumerate(self):
                if i > 0: r += "\n"
                r += str(t)
            return r
        return ''

    def __setitem__(self, key, item):
        self._list[key]=item

    def __getitem__(self, key):
        if key.__class__ == int:
            return self._list[key]

        keyisobj = (type(key) != str and
                    type(key) != unicode)

        for e in self._list:
            if keyisobj:
                if e is key:
                    return e
            else:
                if hasattr(e, 'id'):
                    if e.id == key:   return e
                elif hasattr(e, 'name'):
                    if e.name == key: return e
    @property
    def        first(self):   return  self[0]
    @property
    def        second(self):  return  self[1]
    @property
    def        third(self):   return  self[2]
    @property
    def        fourth(self):  return  self[3]
    @property
    def        fifth(self):   return  self[4]

    @property
    def brokenrules(self):
        return brokenrules()

    @property
    def isvalid(self):
        return self.brokenrules.isempty

class entity():
    def __init__(self):
        pass

    def add(self, t):
        th = entities()
        th += self
        th += t
        return th

    def __add__(self, t):
        return self.add(t)

    @property
    def brokenrules(self):
        return brokenrules()

    @property
    def isvalid(self):
        return self.brokenrules.isempty

<commit_msg>Return index from entities.getrandom()<commit_after># vim: set et ts=4 sw=4 fdm=marker
"""
MIT License

Copyright (c) 2016 Jesse Hogan

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from pdb import set_trace; B=set_trace
from random import randint

class entities(object):
    def __init__(self):
        self.clear()

    def clear(self):
        self._ls=[]

    def __iter__(self):
        for t in self._list:
            yield t

    def getrandom(self):
        if self.isempty: return None
        ix = randint(0, self.ubound)
        return self[ix], ix

    def where(self, fn):
        es = entities()
        for e in self:
            if fn(e): es += e
        return es

    def remove(self, e):
        if callable(e):
            rms = self.where(e)
        else:
            rms = [e]

        for i, e1 in enumerate(self):
            for e2 in rms:
                if e1 is e2:
                    del self._list[i]
                    break

    def reversed(self):
        r = type(self)()
        for e in reversed(self._ls):
            r += e
        return r

    @property
    def ubound(self):
        if self.isempty: return None
        return self.count - 1

    def shift(self):
        return self._ls.pop(0)

    def unshift(self, t):
        # TODO: Return entities object to indicate what was unshifted
        return self._ls.insert(0, t)

    def __lshift__(self, a):
        self.unshift(a)

    def append(self, obj, uniq=False, r=None):
        if not r: r = []
        if isinstance(obj, entity):
            t = obj
        elif isinstance(obj, entities):
            for t in obj:
                if uniq:
                    for t1 in self:
                        if t is t1: break
                    else:
                        self.append(t, r=r)
                        continue
                    break
                else:
                    self.append(t, r=r)
            return r
        else:
            raise ValueError('Unsupported object appended')

        if uniq:
            for t1 in self:
                if t is t1: return r

        r.append(t)
        for t in r:
            self._list.append(t)

        return r

    def __iadd__(self, t):
        self.append(t)
        return self

    def __iand__(self, t):
        self.append(t, uniq=True)
        return self

    def add(self, ts):
        self += ts
        return self

    def __add__(self, t):
        return self.add(t)

    @property
    def _list(self):
        if not hasattr(self, '_ls'):
            self._ls = []
        return self._ls

    @property
    def count(self):
        return len(self._list)

    def __len__(self):
        return self.count

    @property
    def isempty(self):
        return self.count == 0

    @property
    def ispopulated(self):
        return not self.isempty

    def __str__(self):
        if not self.isempty:
            r=''
            for i, t in enumerate(self):
                if i > 0: r += "\n"
                r += str(t)
            return r
        return ''

    def __setitem__(self, key, item):
        self._ls[key]=item

    def __getitem__(self, key):
        if key.__class__ == int:
            return self._list[key]

        keyisobj = (type(key) != str and
                    type(key) != unicode)

        for e in self._list:
            if keyisobj:
                if e is key:
                    return e
            else:
                if hasattr(e, 'id'):
                    if e.id == key:   return e
                elif hasattr(e, 'name'):
                    if e.name == key: return e
    @property
    def        first(self):   return  self[0]
    @property
    def        second(self):  return  self[1]
    @property
    def        third(self):   return  self[2]
    @property
    def        fourth(self):  return  self[3]
    @property
    def        fifth(self):   return  self[4]

    @property
    def brokenrules(self):
        return brokenrules()

    @property
    def isvalid(self):
        return self.brokenrules.isempty

class entity():
    def __init__(self):
        pass

    def add(self, t):
        th = entities()
        th += self
        th += t
        return th

    def __add__(self, t):
        return self.add(t)

    @property
    def brokenrules(self):
        return brokenrules()

    @property
    def isvalid(self):
        return self.brokenrules.isempty

<|endoftext|>


<commit_before>import logging
from optparse import make_option
from django.core.management import BaseCommand
from django.db.models.query import EmptyQuerySet
from open_municipio.monitoring.models import Monitoring
from open_municipio.newscache.models import News
from open_municipio.users.models import UserProfile

class Command(BaseCommand):
    """
    List users that want newsletter messages, with some status details
    - name, email, last received message, next messages to be sent
    """
    help = "List users that want newsletter messages"

    option_list = BaseCommand.option_list

    logger = logging.getLogger('import')

    def handle(self, **options):
        """
        access users profiles an show details for each user wanting the newsletter
        """
        nlprofiles = UserProfile.objects.filter(wants_newsletter=True)
        for profile in nlprofiles:
            mos = Monitoring.objects.filter(user=profile.user)

            # extract the news related to all objects that the user is monitoring
            news = EmptyQuerySet()
            for mo in mos:
                news |= mo.content_object.related_news

            # filter: fetch institutional news of highest priority
            news = news.filter(news_type=News.NEWS_TYPE.institutional).filter(priority=1)

            print '{u.first_name} {u.last_name} ({u.username}) - n_news: {0}'.format(len(news), u=profile.user)
            for notizia in news:
                print u'  - {n.news_date}: {n.text}'.format(n=notizia)

<commit_msg>news are never shown, only counted<commit_after>import logging
from optparse import make_option
from django.core.management import BaseCommand
from django.db.models.query import EmptyQuerySet
from open_municipio.monitoring.models import Monitoring
from open_municipio.newscache.models import News
from open_municipio.users.models import UserProfile

class Command(BaseCommand):
    """
    List users that want newsletter messages, with some status details
    - name, email, last received message, next messages to be sent
    """
    help = "List users that want newsletter messages"

    option_list = BaseCommand.option_list

    logger = logging.getLogger('import')

    def handle(self, **options):
        """
        access users profiles an show details for each user wanting the newsletter
        """
        nlprofiles = UserProfile.objects.filter(wants_newsletter=True)
        for profile in nlprofiles:
            mos = Monitoring.objects.filter(user=profile.user)

            # extract the news related to all objects that the user is monitoring
            news = EmptyQuerySet()
            for mo in mos:
                news |= mo.content_object.related_news

            # filter: fetch institutional news of highest priority
            n_news = news.filter(news_type=News.NEWS_TYPE.institutional).filter(priority=1).count()

            print '{u.first_name} {u.last_name} ({u.username}) - n_news: {0}'.format(n_news, u=profile.user)

<|endoftext|>


<commit_before># Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

{
  'targets': [
    {
      'target_name': 'compiler',
      'type': 'none',
      'dependencies': [
        'compiler/dart-compiler.gyp:dartc',
      ],
      'actions': []
    },
    {
      'target_name': 'runtime',
      'type': 'none',
      'dependencies': [
        'runtime/dart-runtime.gyp:dart',
        'runtime/dart-runtime.gyp:dart_bin',
        'runtime/dart-runtime.gyp:run_vm_tests',
      ],
    },
    {
      'target_name': 'client',
      'type': 'none',
      'dependencies': [
        # TODO(ngeoffray): Fling does not have proper dependencies,
        # so don't build it for now.
        # 'client/dart.gyp:fling',
        'client/dart.gyp:dartserver',
      ],
    },
  ],
}
<commit_msg>Also build process_test to enable running Process tests at the top-level directory. Review URL: http://codereview.chromium.org//8263008<commit_after># Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

{
  'targets': [
    {
      'target_name': 'compiler',
      'type': 'none',
      'dependencies': [
        'compiler/dart-compiler.gyp:dartc',
      ],
      'actions': []
    },
    {
      'target_name': 'runtime',
      'type': 'none',
      'dependencies': [
        'runtime/dart-runtime.gyp:dart',
        'runtime/dart-runtime.gyp:dart_bin',
        'runtime/dart-runtime.gyp:run_vm_tests',
        'runtime/dart-runtime.gyp:process_test',
      ],
    },
    {
      'target_name': 'client',
      'type': 'none',
      'dependencies': [
        # TODO(ngeoffray): Fling does not have proper dependencies,
        # so don't build it for now.
        # 'client/dart.gyp:fling',
        'client/dart.gyp:dartserver',
      ],
    },
  ],
}
<|endoftext|>


<commit_before># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Does a backward pass to output the value of a policy."""
from typing import Optional
from open_spiel.python import policy as policy_std
from open_spiel.python.mfg import distribution as distribution_std
from open_spiel.python.mfg import value
import pyspiel

class PolicyValue(value.ValueFunction):
  """Computes the value of a specified strategy."""

  def __init__(self,
               game,
               distribution: distribution_std.Distribution,
               policy: policy_std.Policy,
               state_value: Optional[value.ValueFunction] = None,
               root_state=None):
    """Initializes the value calculation.

    Args:
      game: The game to analyze.
      distribution: A `distribution.Distribution` object.
      policy: A `policy.Policy` object.
      state_value: A state value function. Defaults to Tabular.
      root_state: The state of the game at which to start. If `None`, the game
        root state is used.
    """
    super(PolicyValue, self).__init__(game)
    if root_state is None:
      self._root_states = game.new_initial_states()
    else:
      self._root_states = [root_state]
    self._distribution = distribution
    self._policy = policy

    self._state_value = (state_value if state_value is not None
                         else value.TabularValueFunction(game))

    self.evaluate()

  def eval_state(self, state):
    """Evaluate the value of a state."""
    state_str = state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID)
    if self._state_value.has(state_str):
      return self._state_value(state_str)
    elif state.is_terminal():
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()])
      return self._state_value(state_str)
    elif state.current_player() == pyspiel.PlayerId.CHANCE:
      self._state_value.set_value(state_str, 0.0)
      for action, prob in state.chance_outcomes():
        new_state = state.child(action)
        self._state_value.add_value(state_str,
                                    prob * self.eval_state(new_state))
      return self._state_value(state_str)
    elif state.current_player() == pyspiel.PlayerId.MEAN_FIELD:
      dist_to_register = state.distribution_support()
      dist = [
          self._distribution.value_str(str_state, 0.)
          for str_state in dist_to_register
      ]
      new_state = state.clone()
      new_state.update_distribution(dist)
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()] +
          self.eval_state(new_state))
      return self._state_value(state_str)
    else:
      assert int(state.current_player()) >= 0, "The player id should be >= 0"
      v = 0.0
      for action, prob in self._policy.action_probabilities(state).items():
        new_state = state.child(action)
        v += prob * self.eval_state(new_state)
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()] + v)
      return self._state_value(state_str)

  def evaluate(self):
    """Evaluate the value over states of self._policy."""
    for state in self._root_states:
      self.eval_state(state)

  def value(self, state, action=None):
    if action is None:
      return self._state_value(
          state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID))
    else:
      new_state = state.child(action)
      return state.rewards()[0] + self._state_value(
          new_state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID))
<commit_msg>Fix nit in policy value value function.<commit_after># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Does a backward pass to output the value of a policy."""
from typing import Optional
from open_spiel.python import policy as policy_std
from open_spiel.python.mfg import distribution as distribution_std
from open_spiel.python.mfg import value
import pyspiel

class PolicyValue(value.ValueFunction):
  """Computes the value of a specified strategy."""

  def __init__(self,
               game,
               distribution: distribution_std.Distribution,
               policy: policy_std.Policy,
               state_value: Optional[value.ValueFunction] = None,
               root_state=None):
    """Initializes the value calculation.

    Args:
      game: The game to analyze.
      distribution: A `distribution.Distribution` object.
      policy: A `policy.Policy` object.
      state_value: A state value function. Defaults to Tabular.
      root_state: The state of the game at which to start. If `None`, the game
        root state is used.
    """
    super(PolicyValue, self).__init__(game)
    if root_state is None:
      self._root_states = game.new_initial_states()
    else:
      self._root_states = [root_state]
    self._distribution = distribution
    self._policy = policy

    self._state_value = (state_value if state_value is not None
                         else value.TabularValueFunction(game))

    self.evaluate()

  def eval_state(self, state):
    """Evaluate the value of a state."""
    state_str = state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID)
    if self._state_value.has(state_str):
      return self._state_value(state_str)
    elif state.is_terminal():
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()])
      return self._state_value(state_str)
    elif state.current_player() == pyspiel.PlayerId.CHANCE:
      self._state_value.set_value(state_str, 0.0)
      for action, prob in state.chance_outcomes():
        new_state = state.child(action)
        self._state_value.add_value(state_str,
                                    prob * self.eval_state(new_state))
      return self._state_value(state_str)
    elif state.current_player() == pyspiel.PlayerId.MEAN_FIELD:
      dist_to_register = state.distribution_support()
      dist = [
          self._distribution.value_str(str_state, 0.)
          for str_state in dist_to_register
      ]
      new_state = state.clone()
      new_state.update_distribution(dist)
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()] +
          self.eval_state(new_state))
      return self._state_value(state_str)
    else:
      assert int(state.current_player()) >= 0, "The player id should be >= 0"
      v = 0.0
      for action, prob in self._policy.action_probabilities(state).items():
        new_state = state.child(action)
        v += prob * self.eval_state(new_state)
      self._state_value.set_value(
          state_str,
          state.rewards()[state.mean_field_population()] + v)
      return self._state_value(state_str)

  def evaluate(self):
    """Evaluate the value over states of self._policy."""
    for state in self._root_states:
      self.eval_state(state)

  def value(self, state, action=None):
    if action is None:
      return self._state_value(
          state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID))
    new_state = state.child(action)
    return state.rewards()[state.mean_field_population()] + self._state_value(
        new_state.observation_string(pyspiel.PlayerId.DEFAULT_PLAYER_ID))
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import math
import timeit
import numpy as np
from PIL import Image
#todo - numpy arrays

class Solver():
    """ FDTD Solver for TM """
    def __init__(self,x_sz =360,y_sz = 360):
        """ Initialize 2d fields arrays """
        self.x_sz = x_sz
        self.y_sz = y_sz
        #allocate 2d arrays for fields
        self.Hx = np.zeros((x_sz+1,y_sz+1))
        self.Hy = np.zeros((x_sz+1,y_sz+1))
        self.Ez = np.zeros((x_sz+1,y_sz+1))

        #allocate 2d arrays for space characteristics

        #free space impedance
        self.imp0 = 377.0
        #courant number
        self.cn = (1.0 / math.sqrt(2))

        #solve step
        self.time = 0

    def _uH(self):
        """ Update H field """
        #space parameters TODO (need to be stored and calculated)
        c1 = 1.0
        c2 = self.cn/self.imp0
        c3 = 1.0
        c4 = self.cn/self.imp0
        # Hx update
        #for i in range(self.x_sz):
        #    for j in range(self.y_sz-1):
        #        self.Hx[i,j] = c1* self.Hx[i,j] - c2*(self.Ez[i,j+1]-self.Ez[i,j])
                # Hy update

        self.Hx[:,:-1] = c1 * self.Hx[:,:-1] - c2*( self.Ez[:,1:] - self.Ez[:,:-1] )
        #for i in range(self.x_sz-1):
        #    for j in range(self.y_sz):
        #        self.Hy[i,j] = c3* self.Hy[i,j] + c4*(self.Ez[i+1,j]-self.Ez[i,j])
        self.Hy[:-1,:] = c3 * self.Hy[:-1,:] + c4 * ( self.Ez[1:,:] - self.Ez[:-1,:] )
    def _uE(self):
        """ Update E field """
        #space parameters TODO
        c5 = 1.0
        c6 = 1.0
        c7 = self.cn*self.imp0

        # Ez update
        #for i in range (1,self.x_sz-1):
        #    for j in range (1,self.y_sz-1):
        #        self.Ez[i,j] = c5*self.Ez[i,j] + c7*((self.Hy[i,j] - self.Hy[i-1,j]) - (self.Hx[i,j] - self.Hx[i,j-1]))

        #numpy vectorization
        self.Ez[1:-1,1:-1] = c5*self.Ez[1:-1,1:-1] + c7*((self.Hy[1:-1,1:-1]-self.Hy[:-2,1:-1]) - (self.Hx[1:-1,1:-1] - self.Hx[1:-1,:-2]))
    def solveStep(self):
        """ Solve single 2D step within gived boundary conditions and source """
        self._uH()
        self._uE()
        self.time += 1
        #TODO Boundary conditions
        #TODO Source
        ppw = 70 #points per wavelength
        a = math.pi* ((self.cn*self.time)/ppw - 1.0)
        a = a*a
        self.Ez[math.floor(self.x_sz/2)][math.floor(self.y_sz/2)] = 10*( 1.0 - 2.0 * a)*math.exp(-a)
        #if self.time < 700:
        #    self.Ez[math.floor(self.x_sz/2)][math.floor(self.y_sz/2)] = math.sin(self.cn*self.time/50)*2

    def solveTime(self,times = 10):
        tmg = [0]*times
        for i in range(times):
            try:
                #preformance measurement for step
                i_t = timeit.default_timer()
                self.solveStep()
                tmg[i] = timeit.default_timer() - i_t
                print("PM:dt=%f,step=%d,xsz=%d,ysz=%d" % (tmg[i],self.time,self.x_sz,self.y_sz))
                #debug dump every 10 step
                #if self.time > 600:
                if self.time % 10 == 0:
                   self.dumpState()
                #except (ValueError):
                #    print ("Simulation reached incorrect values (NaN) at t=%d STOP" % self.time)
            except (OverflowError):
                print ("Simulation reached incorrent values (Too Big) at t=%dSTOP" % self.time)
                return 0
        dt_s = 0.0
        for i in range (times):
            dt_s += tmg[i]
        print ("SOLVED, dt_mean=%f" % (dt_s/i))

    def dumpState(self):
        fname = ("FDTD2D-T%d.png" % self.time)
        img = Image.new("RGB",(self.x_sz,self.y_sz))
        rast = img.load()
        for i in range (self.x_sz):
            for j in range (self.y_sz):
                #TODO do correct remap
                Ep = int (abs(float(self.Ez[i,j])*600))
                #Hp = int (abs((self.Hx[i][j])*600*377 ))
                try:
                    rast[i, j] = (Ep,0,0)
                except (ValueError, OverflowError):
                    rast[i,j] = (255,255,255)
        img.save(fname)
        print ("Snapshot at time %d" % self.time)

#test code
print("START")
slv = Solver()
slv.solveTime(1000)
#slv.dumpState()
<commit_msg>Update FDTDImager.py<commit_after>#!/usr/bin/python
import sys
import math
import timeit
import numpy as np
from PIL import Image
#todo - numpy arrays

class Solver():
    """ FDTD Solver for TM """
    def __init__(self,x_sz =360,y_sz = 360):
        """ Initialize 2d fields arrays """
        self.x_sz = x_sz
        self.y_sz = y_sz
        #allocate 2d arrays for fields
        self.Hx = np.zeros((x_sz+1,y_sz+1))
        self.Hy = np.zeros((x_sz+1,y_sz+1))
        self.Ez = np.zeros((x_sz+1,y_sz+1))

        #allocate 2d arrays for space characteristics

        #free space impedance
        self.imp0 = 377.0
        #courant number
        self.cn = (1.0 / math.sqrt(2))

        #solve step
        self.time = 0

    def _uH(self):
        """ Update H field """
        #space parameters TODO (need to be stored and calculated)
        c1 = 1.0
        c2 = self.cn/self.imp0
        c3 = 1.0
        c4 = self.cn/self.imp0
        # Hx update
        #for i in range(self.x_sz):
        #    for j in range(self.y_sz-1):
        #        self.Hx[i,j] = c1* self.Hx[i,j] - c2*(self.Ez[i,j+1]-self.Ez[i,j])
                # Hy update

        self.Hx[:,:-1] = c1 * self.Hx[:,:-1] - c2*( self.Ez[:,1:] - self.Ez[:,:-1] )
        #for i in range(self.x_sz-1):
        #    for j in range(self.y_sz):
        #        self.Hy[i,j] = c3* self.Hy[i,j] + c4*(self.Ez[i+1,j]-self.Ez[i,j])
        self.Hy[:-1,:] = c3 * self.Hy[:-1,:] + c4 * ( self.Ez[1:,:] - self.Ez[:-1,:] )
    def _uE(self):
        """ Update E field """
        #space parameters TODO
        c5 = 1.0
        c6 = 1.0
        c7 = self.cn*self.imp0

        # Ez update
        #for i in range (1,self.x_sz-1):
        #    for j in range (1,self.y_sz-1):
        #        self.Ez[i,j] = c5*self.Ez[i,j] + c7*((self.Hy[i,j] - self.Hy[i-1,j]) - (self.Hx[i,j] - self.Hx[i,j-1]))

        #numpy vectorization
        self.Ez[1:-1,1:-1] = c5*self.Ez[1:-1,1:-1] + c7*((self.Hy[1:-1,1:-1]-self.Hy[:-2,1:-1]) - (self.Hx[1:-1,1:-1] - self.Hx[1:-1,:-2]))
    def solveStep(self):
        """ Solve single 2D step within gived boundary conditions and source """
        self._uH()
        self._uE()
        self.time += 1
        #TODO Boundary conditions
        #TODO Source
        ppw = 70 #points per wavelength
        a = math.pi* ((self.cn*self.time)/ppw - 1.0)
        a = a*a
        self.Ez[math.floor(self.x_sz/2)][math.floor(self.y_sz/2)] = 10*( 1.0 - 2.0 * a)*math.exp(-a)
        #if self.time < 700:
        #    self.Ez[math.floor(self.x_sz/2)][math.floor(self.y_sz/2)] = math.sin(self.cn*self.time/50)*2

    def solveTime(self,times = 10):
        tmg = [0]*times
        for i in range(times):
            try:
                #preformance measurement for step
                i_t = timeit.default_timer()
                self.solveStep()
                tmg[i] = timeit.default_timer() - i_t
                print("PM:dt=%f,step=%d,xsz=%d,ysz=%d" % (tmg[i],self.time,self.x_sz,self.y_sz))
                #debug dump every 10 step
                #if self.time > 600:
                if self.time % 10 == 0:
                   self.dumpState()
                #except (ValueError):
                #    print ("Simulation reached incorrect values (NaN) at t=%d STOP" % self.time)
            except (OverflowError):
                print ("Simulation reached incorrent values (Too Big) at t=%dSTOP" % self.time)
                return 0
        dt_s = 0.0
        for i in range (times):
            dt_s += tmg[i]
        print ("SOLVED, dt_mean=%f" % (dt_s/i))

    def dumpState(self):
        fname = ("FDTD2D-T%04d.png" % self.time)
        img = Image.new("RGB",(self.x_sz,self.y_sz))
        rast = img.load()
        for i in range (self.x_sz):
            for j in range (self.y_sz):
                #TODO do correct remap
                Ep = int (abs(float(self.Ez[i,j])*600))
                #Hp = int (abs((self.Hx[i][j])*600*377 ))
                try:
                    rast[i, j] = (Ep,0,0)
                except (ValueError, OverflowError):
                    rast[i,j] = (255,255,255)
        img.save(fname)
        print ("Snapshot at time %d" % self.time)

#test code
print("START")
slv = Solver()
slv.solveTime(1000)
#slv.dumpState()
<|endoftext|>


<commit_before>"""Utility function to perform global search and replace in a source tree.

`perlpie` will perform a global search and replace on all files in a
directory recursively.  It's a small python wrapper around the `perl
-p -i -e` functionality, but with easier syntax.  I **strongly
recommend** running `perlpie` on files under source control.  In this
way it's easy to track your changes and if (more likely when) you
discover your regular expression was wrong you can easily revert.  I
also recommend using `grin` to test your regular expressions before
running `perlpie`.

Parameters
----------
oldstring : regular expression
    Regular expression matching the string you want to replace
newstring : string
    The string you would like to replace the oldstring with.  Note
    this is not a regular expression but the exact string. One
    exception to this rule is the at symbol `@`.  This has special
    meaning in perl, so you need an escape character for this.  See
    Examples below.

Requires
--------
perl : The underlying language we're using to perform the search and replace.

`grin <http://pypi.python.org/pypi/grin/>`_ : Grin is a tool written
by Robert Kern to wrap `grep` and `find` with python and easier
command line options.

Example
-------

Replace all occurences of foo with bar::

    perlpie foo bar

Replace numpy.testing with nipy's testing framework::

    perlpie 'from\s+numpy\.testing.*' 'from nipy.testing import *'

Replace all @slow decorators in my code with @dec.super_slow.  Here we
have to escape the @ symbol which has special meaning in perl::

    perlpie '\@slow' '\@dec.super_slow'

Remove all occurences of importing make_doctest_suite::

    perlpie 'from\snipy\.utils\.testutils.*make_doctest_suite'

"""

# notes on perl-dash-pie
# perl -p -i -e 's/oldstring/newstring/g' *
# find . -name '*.html' -print0 | xargs -0 perl -pi -e 's/oldstring/newstring/g'

from optparse import OptionParser
import subprocess

def check_deps():
    try:
        import grin
    except ImportError:
        print 'perlpie requires grin to operate.'
        print 'You can find grin in the python package index:'
        print '  http://pypi.python.org/pypi/grin/'
        return False
    # assume they have perl for now
    return True

def perl_dash_pie(oldstr, newstr):
    """Use perl to replace the oldstr with the newstr.

    Examples
    --------

    # To replace all occurences of 'import numpy as N' with 'import numpy as np'
    from nipy.utils import perlpie
    perlpie.perl_dash_pie('import\s+numpy\s+as\s+N', 'import numpy as np')
    grind | xargs perl -pi -e 's/import\s+numpy\s+as\s+N/import numpy as np/g'

    """

    cmd = "grind | xargs perl -pi -e 's/%s/%s/g'" % (oldstr, newstr)
    print cmd
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError, err:
        msg = """
        Error while executing perl_dash_pie command:
        %s
        Error:
        %s
        """ % (cmd, str(err))
        raise Exception(msg)

def main():
    param_index = __doc__.index('Parameters')
    description = __doc__[:param_index]
    usage = "usage: %prog [options] oldstring newstring"
    parser = OptionParser(usage=usage, description=description)
    (options, args) = parser.parse_args()

    if not args:
        parser.print_help()
        return

    if check_deps():
        oldstr = args[0]
        newstr = args[1]
        perl_dash_pie(oldstr, newstr)
<commit_msg>Cleanup documentation for perlpie script<commit_after>"""Perform a global search and replace on the current directory *recursively*.

This a small python wrapper around the `perl -p -i -e` functionality.
I **strongly recommend** running `perlpie` on files under source
control.  In this way it's easy to track your changes and if you
discover your regular expression was wrong you can easily revert.  I
also recommend using `grin` to test your regular expressions before
running `perlpie`.

Parameters
----------
regex : regular expression
    Regular expression matching the string you want to replace
newstring : string
    The string you would like to replace the oldstring with.  Note
    this is not a regular expression but the exact string. One
    exception to this rule is the at symbol `@`.  This has special
    meaning in perl, so you need an escape character for this.  See
    Examples below.

Requires
--------
perl : The underlying language we're using to perform the search and replace.

`grin <http://pypi.python.org/pypi/grin/>`_ : Grin is a tool written
by Robert Kern to wrap `grep` and `find` with python and easier
command line options.

Examples
--------
Replace all occurences of foo with bar::

    perlpie foo bar

Replace numpy.testing with nipy's testing framework::

    perlpie 'from\s+numpy\.testing.*' 'from nipy.testing import *'

Replace all @slow decorators in my code with @dec.super_slow.  Here we
have to escape the @ symbol which has special meaning in perl::

    perlpie '\@slow' '\@dec.super_slow'

Remove all occurences of importing make_doctest_suite::

    perlpie 'from\snipy\.utils\.testutils.*make_doctest_suite'

"""

# notes on perl-dash-pie
# perl -p -i -e 's/oldstring/newstring/g' *
# find . -name '*.html' -print0 | xargs -0 perl -pi -e 's/oldstring/newstring/g'

from optparse import OptionParser
import subprocess

usage_doc = "usage: %prog [options] regex newstring"

def check_deps():
    try:
        import grin
    except ImportError:
        print 'perlpie requires grin to operate.'
        print 'You can find grin in the python package index:'
        print '  http://pypi.python.org/pypi/grin/'
        return False
    # assume they have perl for now
    return True

def perl_dash_pie(oldstr, newstr):
    """Use perl to replace the oldstr with the newstr.

    Examples
    --------

    # To replace all occurences of 'import numpy as N' with 'import numpy as np'
    from nipy.utils import perlpie
    perlpie.perl_dash_pie('import\s+numpy\s+as\s+N', 'import numpy as np')
    grind | xargs perl -pi -e 's/import\s+numpy\s+as\s+N/import numpy as np/g'

    """

    cmd = "grind | xargs perl -pi -e 's/%s/%s/g'" % (oldstr, newstr)
    print cmd
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError, err:
        msg = """
        Error while executing perl_dash_pie command:
        %s
        Error:
        %s
        """ % (cmd, str(err))
        raise Exception(msg)

def print_extended_help(option, opt_str, value, parser, *args, **kwargs):
    print __doc__

def main():
    description = __doc__.splitlines()[0]
    usage = usage_doc
    parser = OptionParser(usage=usage, description=description)
    parser.add_option('-e', '--extended-help', action='callback',
                      callback=print_extended_help,
                      help='print extended help including examples')
    (options, args) = parser.parse_args()

    if not args:
        parser.print_help()
        return

    if check_deps():
        oldstr = args[0]
        newstr = args[1]
        perl_dash_pie(oldstr, newstr)
<|endoftext|>


<commit_before># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

import unittest

from nix import *

class TestFile(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)

    def tearDown(self):
        self.file.close()

    def test_file_format(self):
        assert(self.file.format == "nix")
        assert(self.file.version == "1.0")

    def test_file_timestamps(self):
        created_at = self.file.created_at
        assert(created_at > 0)

        updated_at = self.file.updated_at
        assert(updated_at > 0)

        self.file.force_created_at(1403530068)
        assert(self.file.created_at == 1403530068)

    def test_file_blocks(self):
        assert(len(self.file.blocks) == 0)

        block = self.file.create_block("test block", "recordingsession")

        assert(len(self.file.blocks) == 1)

        assert(block      in self.file.blocks)
        assert(block.id   in self.file.blocks)
        assert("notexist" not in self.file.blocks)

        assert(block.id == self.file.blocks[0].id)
        assert(block.id == self.file.blocks[-1].id)

        del self.file.blocks[0]

        assert(len(self.file.blocks) == 0)

    def test_file_sections(self):
        assert(len(self.file.sections) == 0)

        section = self.file.create_section("test section", "recordingsession")

        assert(len(self.file.sections) == 1)

        # TODO implement __eq__ for Section
        assert(section      in self.file.sections)
        assert(section.id   in self.file.sections)
        assert("notexist" not in self.file.sections)

        assert(section.id == self.file.sections[0].id)
        assert(section.id == self.file.sections[-1].id)

        del self.file.sections[0]

        assert(len(self.file.sections) == 0)
<commit_msg>test_file.py: remove outdated TODO<commit_after># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

import unittest

from nix import *

class TestFile(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)

    def tearDown(self):
        self.file.close()

    def test_file_format(self):
        assert(self.file.format == "nix")
        assert(self.file.version == "1.0")

    def test_file_timestamps(self):
        created_at = self.file.created_at
        assert(created_at > 0)

        updated_at = self.file.updated_at
        assert(updated_at > 0)

        self.file.force_created_at(1403530068)
        assert(self.file.created_at == 1403530068)

    def test_file_blocks(self):
        assert(len(self.file.blocks) == 0)

        block = self.file.create_block("test block", "recordingsession")

        assert(len(self.file.blocks) == 1)

        assert(block      in self.file.blocks)
        assert(block.id   in self.file.blocks)
        assert("notexist" not in self.file.blocks)

        assert(block.id == self.file.blocks[0].id)
        assert(block.id == self.file.blocks[-1].id)

        del self.file.blocks[0]

        assert(len(self.file.blocks) == 0)

    def test_file_sections(self):
        assert(len(self.file.sections) == 0)

        section = self.file.create_section("test section", "recordingsession")

        assert(len(self.file.sections) == 1)

        assert(section      in self.file.sections)
        assert(section.id   in self.file.sections)
        assert("notexist" not in self.file.sections)

        assert(section.id == self.file.sections[0].id)
        assert(section.id == self.file.sections[-1].id)

        del self.file.sections[0]

        assert(len(self.file.sections) == 0)
<|endoftext|>


<commit_before># Natural Language Toolkit: Interface to Weka Classsifiers
#
# Copyright (C) 2001-2015 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
Classifiers that make use of the external 'Weka' package.
"""
from __future__ import print_function
import time
import tempfile
import os
import subprocess
import re
import zipfile

from sys import stdin
from nltk import compat
from nltk.probability import DictionaryProbDist
from nltk.internals import java, config_java

from nltk.classify.api import ClassifierI

_weka_classpath = None
_weka_search = ['.',
                '/usr/share/weka',
                '/usr/local/share/weka',
                '/usr/lib/weka',
                '/usr/local/lib/weka',]
def config_weka(classpath=None):
    global _weka_classpath

    # Make sure java's configured first.
    config_java()

    if classpath is not None:
        _weka_classpath = classpath

    if _weka_classpath is None:
        searchpath = _weka_search
        if 'WEKAHOME' in os.environ:
            searchpath.insert(0, os.environ['WEKAHOME'])

        for path in searchpath:
            if os.path.exists(os.path.join(path, 'weka.jar')):
                _weka_classpath = os.path.join(path, 'weka.jar')
                version = _check_weka_version(_weka_classpath)
                if version:
                    print(('[Found Weka: %s (version %s)]' %
                           (_weka_classpath, version)))
                else:
                    print('[Found Weka: %s]' % _weka_classpath)
                _check_weka_version(_weka_classpath)

    if _weka_classpath is None:
        raise LookupError('Unable to find weka.jar!  Use config_weka() '
                          'or set the WEKAHOME environment variable. '
                          'For more information about Weka, please see '
                          'http://www.cs.waikato.ac.nz/ml/weka/')

def _check_weka_version(jar):
    try:
        zf = zipfile.ZipFile(jar)
    except SystemExit as KeyboardInterrupt:
        raise
    except:
        return None
    try:
        try:
            return zf.read('weka/core/version.txt')
        except KeyError:
            return None
    finally:
        zf.close()

class WekaClassifier(ClassifierI):
    def __init__(self, formatter, model_filename):
        self._formatter = formatter
        self._model = model_filename

    def prob_classify_many(self, featuresets):
        return self._classify_many(featuresets, ['-p', '0', '-distribution'])

    def classify_many(self, featuresets):
        return self._classify_many(featuresets, ['-p', '0'])

    def _classify_many(self, featuresets, options):
        # Make sure we can find java & weka.
        config_weka()

        temp_dir = tempfile.mkdtemp()
        try:
            # Write the test data file.
            test_filename = os.path.join(temp_dir, 'test.arff')
            self._formatter.write(test_filename, featuresets)

            # Call weka to classify the data.
            cmd = ['weka.classifiers.bayes.NaiveBayes',
                   '-l', self._model, '-T', test_filename] + options
            (stdout, stderr) = java(cmd, classpath=_weka_classpath,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)

            # Check if something went wrong:
            if stderr and not stdout:
                if 'Illegal options: -distribution' in stderr:
                    raise ValueError('The installed version of weka does '
                                     'not support probability distribution '
                                     'output.')
                else:
                    raise ValueError('Weka failed to generate output:\n%s'
                                     % stderr)

            # Parse weka's output.
            return self.parse_weka_output(stdout.decode(stdin.encoding).split('\n'))

        finally:
            for f in os.listdir(temp_dir):
                os.remove(os.path.join(temp_dir, f))
            os.rmdir(temp_dir)

    def parse_weka_distribution(self, s):
        probs = [float(v) for v in re.split('[*,]+', s) if v.strip()]
        probs = dict(zip(self._formatter.labels(), probs))
        return DictionaryProbDist(probs)

    def parse_weka_output(self, lines):
        # Strip unwanted text from stdout
        for i,line in enumerate(lines):
            if line.strip().startswith("inst#"):
                lines = lines[i:]
                break

        if lines[0].split() == ['inst#', 'actual', 'predicted',
                                'error', 'prediction']:
            return [line.split()[2].split(':')[1]
                    for line in lines[1:] if line.strip()]
        elif lines[0].split() == ['inst#', 'actual', 'predicted',
                                  'error', 'distribution']:
            return [self.parse_weka_distribution(line.split()[-1])
                    for line in lines[1:] if line.strip()]

        # is this safe:?
        elif re.match(r'^0 \w+ [01]\.[0-9]* \?\s*$', lines[0]):
            return [line.split()[1] for line in lines if line.strip()]

        else:
            for line in lines[:10]:
                print(line)
            raise ValueError('Unhandled output format -- your version '
                             'of weka may not be supported.\n'
                             '  Header: %s' % lines[0])

    # [xx] full list of classifiers (some may be abstract?):
    # ADTree, AODE, BayesNet, ComplementNaiveBayes, ConjunctiveRule,
    # DecisionStump, DecisionTable, HyperPipes, IB1, IBk, Id3, J48,
    # JRip, KStar, LBR, LeastMedSq, LinearRegression, LMT, Logistic,
    # LogisticBase, M5Base, MultilayerPerceptron,
    # MultipleClassifiersCombiner, NaiveBayes, NaiveBayesMultinomial,
    # NaiveBayesSimple, NBTree, NNge, OneR, PaceRegression, PART,
    # PreConstructedLinearModel, Prism, RandomForest,
    # RandomizableClassifier, RandomTree, RBFNetwork, REPTree, Ridor,
    # RuleNode, SimpleLinearRegression, SimpleLogistic,
    # SingleClassifierEnhancer, SMO, SMOreg, UserClassifier, VFI,
    # VotedPerceptron, Winnow, ZeroR

    _CLASSIFIER_CLASS = {
        'naivebayes': 'weka.classifiers.bayes.NaiveBayes',
        'C4.5': 'weka.classifiers.trees.J48',
        'log_regression': 'weka.classifiers.functions.Logistic',
        'svm': 'weka.classifiers.functions.SMO',
        'kstar': 'weka.classifiers.lazy.KStar',
        'ripper': 'weka.classifiers.rules.JRip',
        }
    @classmethod
    def train(cls, model_filename, featuresets,
              classifier='naivebayes', options=[], quiet=True):
        # Make sure we can find java & weka.
        config_weka()

        # Build an ARFF formatter.
        formatter = ARFF_Formatter.from_train(featuresets)

        temp_dir = tempfile.mkdtemp()
        try:
            # Write the training data file.
            train_filename = os.path.join(temp_dir, 'train.arff')
            formatter.write(train_filename, featuresets)

            if classifier in cls._CLASSIFIER_CLASS:
                javaclass = cls._CLASSIFIER_CLASS[classifier]
            elif classifier in cls._CLASSIFIER_CLASS.values():
                javaclass = classifier
            else:
                raise ValueError('Unknown classifier %s' % classifier)

            # Train the weka model.
            cmd = [javaclass, '-d', model_filename, '-t', train_filename]
            cmd += list(options)
            if quiet:
                stdout = subprocess.PIPE
            else: stdout = None
            java(cmd, classpath=_weka_classpath, stdout=stdout)

            # Return the new classifier.
            return WekaClassifier(formatter, model_filename)

        finally:
            for f in os.listdir(temp_dir):
                os.remove(os.path.join(temp_dir, f))
            os.rmdir(temp_dir)

class ARFF_Formatter:
    """
    Converts featuresets and labeled featuresets to ARFF-formatted
    strings, appropriate for input into Weka.

    Features and classes can be specified manually in the constructor, or may
    be determined from data using ``from_train``.
    """

    def __init__(self, labels, features):
        """
        :param labels: A list of all class labels that can be generated.
        :param features: A list of feature specifications, where
            each feature specification is a tuple (fname, ftype);
            and ftype is an ARFF type string such as NUMERIC or
            STRING.
        """
        self._labels = labels
        self._features = features

    def format(self, tokens):
        """Returns a string representation of ARFF output for the given data."""
        return self.header_section() + self.data_section(tokens)

    def labels(self):
        """Returns the list of classes."""
        return list(self._labels)

    def write(self, outfile, tokens):
        """Writes ARFF data to a file for the given data."""
        if not hasattr(outfile, 'write'):
            outfile = open(outfile, 'w')
        outfile.write(self.format(tokens))
        outfile.close()

    @staticmethod
    def from_train(tokens):
        """
        Constructs an ARFF_Formatter instance with class labels and feature
        types determined from the given data. Handles boolean, numeric and
        string (note: not nominal) types.
        """
        # Find the set of all attested labels.
        labels = set(label for (tok, label) in tokens)

        # Determine the types of all features.
        features = {}
        for tok, label in tokens:
            for (fname, fval) in tok.items():
                if issubclass(type(fval), bool):
                    ftype = '{True, False}'
                elif issubclass(type(fval), (compat.integer_types, float, bool)):
                    ftype = 'NUMERIC'
                elif issubclass(type(fval), compat.string_types):
                    ftype = 'STRING'
                elif fval is None:
                    continue # can't tell the type.
                else:
                    raise ValueError('Unsupported value type %r' % ftype)

                if features.get(fname, ftype) != ftype:
                    raise ValueError('Inconsistent type for %s' % fname)
                features[fname] = ftype
        features = sorted(features.items())

        return ARFF_Formatter(labels, features)

    def header_section(self):
        """Returns an ARFF header as a string."""
        # Header comment.
        s = ('% Weka ARFF file\n' +
             '% Generated automatically by NLTK\n' +
             '%% %s\n\n' % time.ctime())

        # Relation name
        s += '@RELATION rel\n\n'

        # Input attribute specifications
        for fname, ftype in self._features:
            s += '@ATTRIBUTE %-30r %s\n' % (fname, ftype)

        # Label attribute specification
        s += '@ATTRIBUTE %-30r {%s}\n' % ('-label-', ','.join(self._labels))

        return s

    def data_section(self, tokens, labeled=None):
        """
        Returns the ARFF data section for the given data.

        :param tokens: a list of featuresets (dicts) or labelled featuresets
            which are tuples (featureset, label).
        :param labeled: Indicates whether the given tokens are labeled
            or not.  If None, then the tokens will be assumed to be
            labeled if the first token's value is a tuple or list.
        """
        # Check if the tokens are labeled or unlabeled.  If unlabeled,
        # then use 'None'
        if labeled is None:
            labeled = tokens and isinstance(tokens[0], (tuple, list))
        if not labeled:
            tokens = [(tok, None) for tok in tokens]

        # Data section
        s = '\n@DATA\n'
        for (tok, label) in tokens:
            for fname, ftype in self._features:
                s += '%s,' % self._fmt_arff_val(tok.get(fname))
            s += '%s\n' % self._fmt_arff_val(label)

        return s

    def _fmt_arff_val(self, fval):
        if fval is None:
            return '?'
        elif isinstance(fval, (bool, compat.integer_types)):
            return '%s' % fval
        elif isinstance(fval, float):
            return '%r' % fval
        else:
            return '%r' % fval

if __name__ == '__main__':
    from nltk.classify.util import names_demo, binary_names_demo_features
    def make_classifier(featuresets):
        return WekaClassifier.train('/tmp/name.model', featuresets,
                                    'C4.5')
    classifier = names_demo(make_classifier, binary_names_demo_features)
<commit_msg>Ensure that proper exceptions are caught correctly<commit_after># Natural Language Toolkit: Interface to Weka Classsifiers
#
# Copyright (C) 2001-2015 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
Classifiers that make use of the external 'Weka' package.
"""
from __future__ import print_function
import time
import tempfile
import os
import subprocess
import re
import zipfile

from sys import stdin
from nltk import compat
from nltk.probability import DictionaryProbDist
from nltk.internals import java, config_java

from nltk.classify.api import ClassifierI

_weka_classpath = None
_weka_search = ['.',
                '/usr/share/weka',
                '/usr/local/share/weka',
                '/usr/lib/weka',
                '/usr/local/lib/weka',]
def config_weka(classpath=None):
    global _weka_classpath

    # Make sure java's configured first.
    config_java()

    if classpath is not None:
        _weka_classpath = classpath

    if _weka_classpath is None:
        searchpath = _weka_search
        if 'WEKAHOME' in os.environ:
            searchpath.insert(0, os.environ['WEKAHOME'])

        for path in searchpath:
            if os.path.exists(os.path.join(path, 'weka.jar')):
                _weka_classpath = os.path.join(path, 'weka.jar')
                version = _check_weka_version(_weka_classpath)
                if version:
                    print(('[Found Weka: %s (version %s)]' %
                           (_weka_classpath, version)))
                else:
                    print('[Found Weka: %s]' % _weka_classpath)
                _check_weka_version(_weka_classpath)

    if _weka_classpath is None:
        raise LookupError('Unable to find weka.jar!  Use config_weka() '
                          'or set the WEKAHOME environment variable. '
                          'For more information about Weka, please see '
                          'http://www.cs.waikato.ac.nz/ml/weka/')

def _check_weka_version(jar):
    try:
        zf = zipfile.ZipFile(jar)
    except (SystemExit, KeyboardInterrupt):
        raise
    except:
        return None
    try:
        try:
            return zf.read('weka/core/version.txt')
        except KeyError:
            return None
    finally:
        zf.close()

class WekaClassifier(ClassifierI):
    def __init__(self, formatter, model_filename):
        self._formatter = formatter
        self._model = model_filename

    def prob_classify_many(self, featuresets):
        return self._classify_many(featuresets, ['-p', '0', '-distribution'])

    def classify_many(self, featuresets):
        return self._classify_many(featuresets, ['-p', '0'])

    def _classify_many(self, featuresets, options):
        # Make sure we can find java & weka.
        config_weka()

        temp_dir = tempfile.mkdtemp()
        try:
            # Write the test data file.
            test_filename = os.path.join(temp_dir, 'test.arff')
            self._formatter.write(test_filename, featuresets)

            # Call weka to classify the data.
            cmd = ['weka.classifiers.bayes.NaiveBayes',
                   '-l', self._model, '-T', test_filename] + options
            (stdout, stderr) = java(cmd, classpath=_weka_classpath,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)

            # Check if something went wrong:
            if stderr and not stdout:
                if 'Illegal options: -distribution' in stderr:
                    raise ValueError('The installed version of weka does '
                                     'not support probability distribution '
                                     'output.')
                else:
                    raise ValueError('Weka failed to generate output:\n%s'
                                     % stderr)

            # Parse weka's output.
            return self.parse_weka_output(stdout.decode(stdin.encoding).split('\n'))

        finally:
            for f in os.listdir(temp_dir):
                os.remove(os.path.join(temp_dir, f))
            os.rmdir(temp_dir)

    def parse_weka_distribution(self, s):
        probs = [float(v) for v in re.split('[*,]+', s) if v.strip()]
        probs = dict(zip(self._formatter.labels(), probs))
        return DictionaryProbDist(probs)

    def parse_weka_output(self, lines):
        # Strip unwanted text from stdout
        for i,line in enumerate(lines):
            if line.strip().startswith("inst#"):
                lines = lines[i:]
                break

        if lines[0].split() == ['inst#', 'actual', 'predicted',
                                'error', 'prediction']:
            return [line.split()[2].split(':')[1]
                    for line in lines[1:] if line.strip()]
        elif lines[0].split() == ['inst#', 'actual', 'predicted',
                                  'error', 'distribution']:
            return [self.parse_weka_distribution(line.split()[-1])
                    for line in lines[1:] if line.strip()]

        # is this safe:?
        elif re.match(r'^0 \w+ [01]\.[0-9]* \?\s*$', lines[0]):
            return [line.split()[1] for line in lines if line.strip()]

        else:
            for line in lines[:10]:
                print(line)
            raise ValueError('Unhandled output format -- your version '
                             'of weka may not be supported.\n'
                             '  Header: %s' % lines[0])

    # [xx] full list of classifiers (some may be abstract?):
    # ADTree, AODE, BayesNet, ComplementNaiveBayes, ConjunctiveRule,
    # DecisionStump, DecisionTable, HyperPipes, IB1, IBk, Id3, J48,
    # JRip, KStar, LBR, LeastMedSq, LinearRegression, LMT, Logistic,
    # LogisticBase, M5Base, MultilayerPerceptron,
    # MultipleClassifiersCombiner, NaiveBayes, NaiveBayesMultinomial,
    # NaiveBayesSimple, NBTree, NNge, OneR, PaceRegression, PART,
    # PreConstructedLinearModel, Prism, RandomForest,
    # RandomizableClassifier, RandomTree, RBFNetwork, REPTree, Ridor,
    # RuleNode, SimpleLinearRegression, SimpleLogistic,
    # SingleClassifierEnhancer, SMO, SMOreg, UserClassifier, VFI,
    # VotedPerceptron, Winnow, ZeroR

    _CLASSIFIER_CLASS = {
        'naivebayes': 'weka.classifiers.bayes.NaiveBayes',
        'C4.5': 'weka.classifiers.trees.J48',
        'log_regression': 'weka.classifiers.functions.Logistic',
        'svm': 'weka.classifiers.functions.SMO',
        'kstar': 'weka.classifiers.lazy.KStar',
        'ripper': 'weka.classifiers.rules.JRip',
        }
    @classmethod
    def train(cls, model_filename, featuresets,
              classifier='naivebayes', options=[], quiet=True):
        # Make sure we can find java & weka.
        config_weka()

        # Build an ARFF formatter.
        formatter = ARFF_Formatter.from_train(featuresets)

        temp_dir = tempfile.mkdtemp()
        try:
            # Write the training data file.
            train_filename = os.path.join(temp_dir, 'train.arff')
            formatter.write(train_filename, featuresets)

            if classifier in cls._CLASSIFIER_CLASS:
                javaclass = cls._CLASSIFIER_CLASS[classifier]
            elif classifier in cls._CLASSIFIER_CLASS.values():
                javaclass = classifier
            else:
                raise ValueError('Unknown classifier %s' % classifier)

            # Train the weka model.
            cmd = [javaclass, '-d', model_filename, '-t', train_filename]
            cmd += list(options)
            if quiet:
                stdout = subprocess.PIPE
            else: stdout = None
            java(cmd, classpath=_weka_classpath, stdout=stdout)

            # Return the new classifier.
            return WekaClassifier(formatter, model_filename)

        finally:
            for f in os.listdir(temp_dir):
                os.remove(os.path.join(temp_dir, f))
            os.rmdir(temp_dir)

class ARFF_Formatter:
    """
    Converts featuresets and labeled featuresets to ARFF-formatted
    strings, appropriate for input into Weka.

    Features and classes can be specified manually in the constructor, or may
    be determined from data using ``from_train``.
    """

    def __init__(self, labels, features):
        """
        :param labels: A list of all class labels that can be generated.
        :param features: A list of feature specifications, where
            each feature specification is a tuple (fname, ftype);
            and ftype is an ARFF type string such as NUMERIC or
            STRING.
        """
        self._labels = labels
        self._features = features

    def format(self, tokens):
        """Returns a string representation of ARFF output for the given data."""
        return self.header_section() + self.data_section(tokens)

    def labels(self):
        """Returns the list of classes."""
        return list(self._labels)

    def write(self, outfile, tokens):
        """Writes ARFF data to a file for the given data."""
        if not hasattr(outfile, 'write'):
            outfile = open(outfile, 'w')
        outfile.write(self.format(tokens))
        outfile.close()

    @staticmethod
    def from_train(tokens):
        """
        Constructs an ARFF_Formatter instance with class labels and feature
        types determined from the given data. Handles boolean, numeric and
        string (note: not nominal) types.
        """
        # Find the set of all attested labels.
        labels = set(label for (tok, label) in tokens)

        # Determine the types of all features.
        features = {}
        for tok, label in tokens:
            for (fname, fval) in tok.items():
                if issubclass(type(fval), bool):
                    ftype = '{True, False}'
                elif issubclass(type(fval), (compat.integer_types, float, bool)):
                    ftype = 'NUMERIC'
                elif issubclass(type(fval), compat.string_types):
                    ftype = 'STRING'
                elif fval is None:
                    continue # can't tell the type.
                else:
                    raise ValueError('Unsupported value type %r' % ftype)

                if features.get(fname, ftype) != ftype:
                    raise ValueError('Inconsistent type for %s' % fname)
                features[fname] = ftype
        features = sorted(features.items())

        return ARFF_Formatter(labels, features)

    def header_section(self):
        """Returns an ARFF header as a string."""
        # Header comment.
        s = ('% Weka ARFF file\n' +
             '% Generated automatically by NLTK\n' +
             '%% %s\n\n' % time.ctime())

        # Relation name
        s += '@RELATION rel\n\n'

        # Input attribute specifications
        for fname, ftype in self._features:
            s += '@ATTRIBUTE %-30r %s\n' % (fname, ftype)

        # Label attribute specification
        s += '@ATTRIBUTE %-30r {%s}\n' % ('-label-', ','.join(self._labels))

        return s

    def data_section(self, tokens, labeled=None):
        """
        Returns the ARFF data section for the given data.

        :param tokens: a list of featuresets (dicts) or labelled featuresets
            which are tuples (featureset, label).
        :param labeled: Indicates whether the given tokens are labeled
            or not.  If None, then the tokens will be assumed to be
            labeled if the first token's value is a tuple or list.
        """
        # Check if the tokens are labeled or unlabeled.  If unlabeled,
        # then use 'None'
        if labeled is None:
            labeled = tokens and isinstance(tokens[0], (tuple, list))
        if not labeled:
            tokens = [(tok, None) for tok in tokens]

        # Data section
        s = '\n@DATA\n'
        for (tok, label) in tokens:
            for fname, ftype in self._features:
                s += '%s,' % self._fmt_arff_val(tok.get(fname))
            s += '%s\n' % self._fmt_arff_val(label)

        return s

    def _fmt_arff_val(self, fval):
        if fval is None:
            return '?'
        elif isinstance(fval, (bool, compat.integer_types)):
            return '%s' % fval
        elif isinstance(fval, float):
            return '%r' % fval
        else:
            return '%r' % fval

if __name__ == '__main__':
    from nltk.classify.util import names_demo, binary_names_demo_features
    def make_classifier(featuresets):
        return WekaClassifier.train('/tmp/name.model', featuresets,
                                    'C4.5')
    classifier = names_demo(make_classifier, binary_names_demo_features)
<|endoftext|>


<commit_before>'''
    nltools Nipype Interfaces
    =========================

    Classes for various nipype interfaces

'''

__all__ = ['Plot_Coregistration_Montage', 'PlotRealignmentParameters', 'Create_Covariates']
__author__ = ["Luke Chang"]
__license__ = "MIT"

# import matplotlib
# matplotlib.use('Agg')
import pandas as pd
import numpy as np
# import pylab as plt
import os
import nibabel as nib
from nipype.interfaces.base import isdefined, BaseInterface, TraitedSpec, File, traits
from nilearn import plotting, image
import nibabel as nib

class Plot_Coregistration_Montage_InputSpec(TraitedSpec):
	wra_img = File(exists=True, mandatory=True)
	canonical_img = File(exists=True, mandatory=True)
	title = traits.Str("Normalized Functional Check", usedefault=True)

class Plot_Coregistration_Montage_OutputSpec(TraitedSpec):
	plot = File(exists=True)

class Plot_Coregistration_Montage(BaseInterface):
	# This function creates an axial montage of the average normalized functional data
	# and overlays outline of the normalized single subject overlay.
	# Could probably pick a better overlay later.

	input_spec = Plot_Coregistration_Montage_InputSpec
	output_spec = Plot_Coregistration_Montage_OutputSpec

	def _run_interface(self, runtime):
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt

		wra_img = nib.load(self.inputs.wra_img)
		canonical_img = nib.load(self.inputs.canonical_img)
		title = self.inputs.title
		mean_wraimg = image.mean_img(wra_img)

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "plot.pdf"

		fig = plotting.plot_anat(mean_wraimg, title="wrafunc & canonical single subject", cut_coords=range(-40, 40, 10), display_mode='z')
		fig.add_edges(canonical_img)
		fig.savefig(filename)
		fig.close()

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class Plot_Quality_Control_InputSpec(TraitedSpec):
	dat_img = File(exists=True, mandatory=True)
	title = traits.Str("Quality Control Plot", usedefault=True)

class Plot_Quality_Control_OutputSpec(TraitedSpec):
	plot = File(exists=True)

class Plot_Quality_Control(BaseInterface):
	# This function creates quality control plots for a 4D time series.
	# Recommend running this after realignment.

	input_spec = Plot_Quality_Control_InputSpec
	output_spec = Plot_Quality_Control_OutputSpec

	def _run_interface(self, runtime):
		from __future__ import division
		import numpy as np
		import nibabel as nib
		from nilearn.masking import compute_epi_mask, apply_mask
		from nilearn.plotting import plot_stat_map
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt

		dat_img = nib.load(self.inputs.dat_img)
		mn = nib.Nifti1Image(np.mean(dat_img.get_data(),axis=3),affine=dat_img.get_affine())
		sd = nib.Nifti1Image(np.std(dat_img.get_data(),axis=3),affine=dat_img.get_affine())
		snr = nib.Nifti1Image(mn.get_data()/sd.get_data(),affine=dat_img.get_affine())
		mask = compute_epi_mask(dat_img)
		masked_data = apply_mask(dat_img, mask)
		global_mn = np.mean(masked_data,axis=1)
		global_sd = np.std(masked_data,axis=1)
		global_outlier = np.append(np.where(global_mn>np.mean(global_mn)+np.std(global_mn)*3),
		                           np.where(global_mn<np.mean(global_mn)-np.std(global_mn)*3))
		frame_diff = np.mean(np.abs(np.diff(masked_data,axis=0)),axis=1)
		frame_outlier = np.append(np.where(frame_diff>np.mean(frame_diff)+np.std(frame_diff)*3),
		                           np.where(frame_diff<np.mean(frame_diff)-np.std(frame_diff)*3))

		title = self.inputs.title

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "Quality_Control_Plot.pdf"

		f, ax = plt.subplots(6,figsize=(15,15))
		plot_stat_map(mn, title="Mean",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[0],
		              draw_cross=False, black_bg='True',annotate=False,)

		plot_stat_map(sd, title="Standard Deviation",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[1],
		              draw_cross=False, black_bg='True',annotate=False,)

		plot_stat_map(snr, title="SNR (mn/sd)",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[2],
		              draw_cross=False, black_bg='True',annotate=False,)

		ax[3].plot(global_mn)
		# ax[3].set_title('Average Signal Intensity')
		ax[3].set_xlabel('TR')
		ax[3].set_ylabel('Global Signal Mean')
		for x in global_outlier:
		    ax[3].axvline(x, color='r', linestyle='--')

		ax[4].plot(global_sd)
		# ax[4].set_title('Frame Differencing')
		ax[4].set_xlabel('TR')
		ax[4].set_ylabel('Global Signal Std')

		ax[5].plot(frame_diff)
		# ax[4].set_title('Frame Differencing')
		ax[5].set_xlabel('TR')
		ax[5].set_ylabel('Avg Abs Diff')
		for x in frame_outlier:
		    ax[5].axvline(x, color='r', linestyle='--')
		f.savefig(filename)
		f.close()

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class PlotRealignmentParametersInputSpec(TraitedSpec):
	realignment_parameters = File(exists=True, mandatory=True)
	outlier_files = File(exists=True)
	title = traits.Str("Realignment parameters", usedefault=True)
	dpi = traits.Int(300, usedefault = True)

class PlotRealignmentParametersOutputSpec(TraitedSpec):
	plot = File(exists=True)

class PlotRealignmentParameters(BaseInterface):
	#This function is adapted from Chris Gorgolewski and creates a figure of the realignment parameters

	input_spec = PlotRealignmentParametersInputSpec
	output_spec = PlotRealignmentParametersOutputSpec

	def _run_interface(self, runtime):
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt
		realignment_parameters = np.loadtxt(self.inputs.realignment_parameters)
		title = self.inputs.title

		F = plt.figure(figsize=(8.3,11.7))
		F.text(0.5, 0.96, self.inputs.title, horizontalalignment='center')
		ax1 = plt.subplot2grid((2,2),(0,0), colspan=2)
		handles =ax1.plot(realignment_parameters[:,0:3])
		ax1.legend(handles, ["x translation", "y translation", "z translation"], loc=0)
		ax1.set_xlabel("image #")
		ax1.set_ylabel("mm")
		ax1.set_xlim((0,realignment_parameters.shape[0]-1))
		ax1.set_ylim(bottom = realignment_parameters[:,0:3].min(), top = realignment_parameters[:,0:3].max())

		ax2 = plt.subplot2grid((2,2),(1,0), colspan=2)
		handles= ax2.plot(realignment_parameters[:,3:6]*180.0/np.pi)
		ax2.legend(handles, ["pitch", "roll", "yaw"], loc=0)
		ax2.set_xlabel("image #")
		ax2.set_ylabel("degrees")
		ax2.set_xlim((0,realignment_parameters.shape[0]-1))
		ax2.set_ylim(bottom=(realignment_parameters[:,3:6]*180.0/np.pi).min(), top= (realignment_parameters[:,3:6]*180.0/np.pi).max())

		if isdefined(self.inputs.outlier_files):
			try:
				outliers = np.loadtxt(self.inputs.outlier_files)
			except IOError as e:
				if e.args[0] == "End-of-file reached before encountering data.":
					pass
				else:
					raise
			else:
				if outliers.size > 0:
					ax1.vlines(outliers, ax1.get_ylim()[0], ax1.get_ylim()[1])
					ax2.vlines(outliers, ax2.get_ylim()[0], ax2.get_ylim()[1])

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "plot.pdf"

		F.savefig(filename,papertype="a4",dpi=self.inputs.dpi)
		plt.clf()
		plt.close()
		del F

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class Create_Covariates_InputSpec(TraitedSpec):
	realignment_parameters = File(exists=True, mandatory=True)
	spike_id = File(exists=True, mandatory=True)

class Create_Covariates_OutputSpec(TraitedSpec):
	covariates = File(exists=True)

class Create_Covariates(BaseInterface):
	input_spec = Create_Covariates_InputSpec
	output_spec = Create_Covariates_OutputSpec

	def _run_interface(self, runtime):
		ra = pd.read_table(self.inputs.realignment_parameters, header=None, sep=r"\s*", names=['ra' + str(x) for x in range(1,7)])
		spike = pd.read_table(self.inputs.spike_id, header=None,names=['Spikes'])

		ra = ra-ra.mean() #mean center
		ra[['rasq' + str(x) for x in range(1,7)]] = ra**2 #add squared
		ra[['radiff' + str(x) for x in range(1,7)]] = pd.DataFrame(ra[ra.columns[0:6]].diff()) #derivative
		ra[['radiffsq' + str(x) for x in range(1,7)]] = pd.DataFrame(ra[ra.columns[0:6]].diff())**2 #derivatives squared

		#build spike regressors
		for i,loc in enumerate(spike['Spikes']):
			ra['spike' + str(i+1)] = 0
			ra['spike' + str(i+1)].iloc[loc] = 1

		filename = 'covariates.csv'
		ra.to_csv(filename, index=False) #write out to file
		self._covariates = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["covariates"] = os.path.abspath(self._covariates)
		return outputs

<commit_msg>made SD cutoff variable input on quality control plot<commit_after>'''
    nltools Nipype Interfaces
    =========================

    Classes for various nipype interfaces

'''

__all__ = ['Plot_Coregistration_Montage', 'PlotRealignmentParameters', 'Create_Covariates']
__author__ = ["Luke Chang"]
__license__ = "MIT"

# import matplotlib
# matplotlib.use('Agg')
import pandas as pd
import numpy as np
# import pylab as plt
import os
import nibabel as nib
from nipype.interfaces.base import isdefined, BaseInterface, TraitedSpec, File, traits
from nilearn import plotting, image
import nibabel as nib

class Plot_Coregistration_Montage_InputSpec(TraitedSpec):
	wra_img = File(exists=True, mandatory=True)
	canonical_img = File(exists=True, mandatory=True)
	title = traits.Str("Normalized Functional Check", usedefault=True)

class Plot_Coregistration_Montage_OutputSpec(TraitedSpec):
	plot = File(exists=True)

class Plot_Coregistration_Montage(BaseInterface):
	# This function creates an axial montage of the average normalized functional data
	# and overlays outline of the normalized single subject overlay.
	# Could probably pick a better overlay later.

	input_spec = Plot_Coregistration_Montage_InputSpec
	output_spec = Plot_Coregistration_Montage_OutputSpec

	def _run_interface(self, runtime):
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt

		wra_img = nib.load(self.inputs.wra_img)
		canonical_img = nib.load(self.inputs.canonical_img)
		title = self.inputs.title
		mean_wraimg = image.mean_img(wra_img)

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "plot.pdf"

		fig = plotting.plot_anat(mean_wraimg, title="wrafunc & canonical single subject", cut_coords=range(-40, 40, 10), display_mode='z')
		fig.add_edges(canonical_img)
		fig.savefig(filename)
		fig.close()

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class Plot_Quality_Control_InputSpec(TraitedSpec):
	dat_img = File(exists=True, mandatory=True)
	title = traits.Str("Quality Control Plot", usedefault=True)
	global_outlier_cutoff = traits.Float(3, usedefault=True)
	frame_outlier_cutoff = traits.Float(3, usedefault=True)

class Plot_Quality_Control_OutputSpec(TraitedSpec):
	plot = File(exists=True)

class Plot_Quality_Control(BaseInterface):
	# This function creates quality control plots for a 4D time series.
	# Recommend running this after realignment.

	input_spec = Plot_Quality_Control_InputSpec
	output_spec = Plot_Quality_Control_OutputSpec

	def _run_interface(self, runtime):
		from __future__ import division
		import numpy as np
		import nibabel as nib
		from nilearn.masking import compute_epi_mask, apply_mask
		from nilearn.plotting import plot_stat_map
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt

		dat_img = nib.load(self.inputs.dat_img)
		mn = nib.Nifti1Image(np.mean(dat_img.get_data(),axis=3),affine=dat_img.get_affine())
		sd = nib.Nifti1Image(np.std(dat_img.get_data(),axis=3),affine=dat_img.get_affine())
		snr = nib.Nifti1Image(mn.get_data()/sd.get_data(),affine=dat_img.get_affine())
		mask = compute_epi_mask(dat_img)
		masked_data = apply_mask(dat_img, mask)
		global_mn = np.mean(masked_data,axis=1)
		global_sd = np.std(masked_data,axis=1)
		global_outlier = np.append(np.where(global_mn>np.mean(global_mn)+np.std(global_mn)*global_outlier_cutoff),
		                           np.where(global_mn<np.mean(global_mn)-np.std(global_mn)*global_outlier_cutoff))
		frame_diff = np.mean(np.abs(np.diff(masked_data,axis=0)),axis=1)
		frame_outlier = np.append(np.where(frame_diff>np.mean(frame_diff)+np.std(frame_diff)*frame_outlier_cutoff),
		                           np.where(frame_diff<np.mean(frame_diff)-np.std(frame_diff)*frame_outlier_cutoff))

		title = self.inputs.title

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "Quality_Control_Plot.pdf"

		f, ax = plt.subplots(6,figsize=(15,15))
		plot_stat_map(mn, title="Mean",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[0],
		              draw_cross=False, black_bg='True',annotate=False,)

		plot_stat_map(sd, title="Standard Deviation",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[1],
		              draw_cross=False, black_bg='True',annotate=False,)

		plot_stat_map(snr, title="SNR (mn/sd)",cut_coords=range(-40, 40, 10), display_mode='z',axes=ax[2],
		              draw_cross=False, black_bg='True',annotate=False,)

		ax[3].plot(global_mn)
		# ax[3].set_title('Average Signal Intensity')
		ax[3].set_xlabel('TR')
		ax[3].set_ylabel('Global Signal Mean')
		for x in global_outlier:
		    ax[3].axvline(x, color='r', linestyle='--')

		ax[4].plot(global_sd)
		# ax[4].set_title('Frame Differencing')
		ax[4].set_xlabel('TR')
		ax[4].set_ylabel('Global Signal Std')

		ax[5].plot(frame_diff)
		# ax[4].set_title('Frame Differencing')
		ax[5].set_xlabel('TR')
		ax[5].set_ylabel('Avg Abs Diff')
		for x in frame_outlier:
		    ax[5].axvline(x, color='r', linestyle='--')
		f.savefig(filename)
		f.close()

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class PlotRealignmentParametersInputSpec(TraitedSpec):
	realignment_parameters = File(exists=True, mandatory=True)
	outlier_files = File(exists=True)
	title = traits.Str("Realignment parameters", usedefault=True)
	dpi = traits.Int(300, usedefault = True)

class PlotRealignmentParametersOutputSpec(TraitedSpec):
	plot = File(exists=True)

class PlotRealignmentParameters(BaseInterface):
	#This function is adapted from Chris Gorgolewski and creates a figure of the realignment parameters

	input_spec = PlotRealignmentParametersInputSpec
	output_spec = PlotRealignmentParametersOutputSpec

	def _run_interface(self, runtime):
		import matplotlib
		matplotlib.use('Agg')
		import pylab as plt
		realignment_parameters = np.loadtxt(self.inputs.realignment_parameters)
		title = self.inputs.title

		F = plt.figure(figsize=(8.3,11.7))
		F.text(0.5, 0.96, self.inputs.title, horizontalalignment='center')
		ax1 = plt.subplot2grid((2,2),(0,0), colspan=2)
		handles =ax1.plot(realignment_parameters[:,0:3])
		ax1.legend(handles, ["x translation", "y translation", "z translation"], loc=0)
		ax1.set_xlabel("image #")
		ax1.set_ylabel("mm")
		ax1.set_xlim((0,realignment_parameters.shape[0]-1))
		ax1.set_ylim(bottom = realignment_parameters[:,0:3].min(), top = realignment_parameters[:,0:3].max())

		ax2 = plt.subplot2grid((2,2),(1,0), colspan=2)
		handles= ax2.plot(realignment_parameters[:,3:6]*180.0/np.pi)
		ax2.legend(handles, ["pitch", "roll", "yaw"], loc=0)
		ax2.set_xlabel("image #")
		ax2.set_ylabel("degrees")
		ax2.set_xlim((0,realignment_parameters.shape[0]-1))
		ax2.set_ylim(bottom=(realignment_parameters[:,3:6]*180.0/np.pi).min(), top= (realignment_parameters[:,3:6]*180.0/np.pi).max())

		if isdefined(self.inputs.outlier_files):
			try:
				outliers = np.loadtxt(self.inputs.outlier_files)
			except IOError as e:
				if e.args[0] == "End-of-file reached before encountering data.":
					pass
				else:
					raise
			else:
				if outliers.size > 0:
					ax1.vlines(outliers, ax1.get_ylim()[0], ax1.get_ylim()[1])
					ax2.vlines(outliers, ax2.get_ylim()[0], ax2.get_ylim()[1])

		if title != "":
			filename = title.replace(" ", "_")+".pdf"
		else:
			filename = "plot.pdf"

		F.savefig(filename,papertype="a4",dpi=self.inputs.dpi)
		plt.clf()
		plt.close()
		del F

		self._plot = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["plot"] = os.path.abspath(self._plot)
		return outputs

class Create_Covariates_InputSpec(TraitedSpec):
	realignment_parameters = File(exists=True, mandatory=True)
	spike_id = File(exists=True, mandatory=True)

class Create_Covariates_OutputSpec(TraitedSpec):
	covariates = File(exists=True)

class Create_Covariates(BaseInterface):
	input_spec = Create_Covariates_InputSpec
	output_spec = Create_Covariates_OutputSpec

	def _run_interface(self, runtime):
		ra = pd.read_table(self.inputs.realignment_parameters, header=None, sep=r"\s*", names=['ra' + str(x) for x in range(1,7)])
		spike = pd.read_table(self.inputs.spike_id, header=None,names=['Spikes'])

		ra = ra-ra.mean() #mean center
		ra[['rasq' + str(x) for x in range(1,7)]] = ra**2 #add squared
		ra[['radiff' + str(x) for x in range(1,7)]] = pd.DataFrame(ra[ra.columns[0:6]].diff()) #derivative
		ra[['radiffsq' + str(x) for x in range(1,7)]] = pd.DataFrame(ra[ra.columns[0:6]].diff())**2 #derivatives squared

		#build spike regressors
		for i,loc in enumerate(spike['Spikes']):
			ra['spike' + str(i+1)] = 0
			ra['spike' + str(i+1)].iloc[loc] = 1

		filename = 'covariates.csv'
		ra.to_csv(filename, index=False) #write out to file
		self._covariates = filename

		runtime.returncode=0
		return runtime

	def _list_outputs(self):
		outputs = self._outputs().get()
		outputs["covariates"] = os.path.abspath(self._covariates)
		return outputs

<|endoftext|>


<commit_before>'''
Google Authenticator API
------------------------

Google Authenticator is based on HOTP and TOTP. It provides a simple way of
provisionning an OTP generator through a new URL scheme.

This module provides parsing and high-level API over the classic HOTP and TOTP
APIs provided by the oath.hotp and oath.totp modules.
'''

import re
import urlparse
import base64
import hashlib
import urllib

from . import _hotp as hotp
from . import _totp as totp

__all__ = ('GoogleAuthenticator', 'from_b32key')

otpauth_re = re.compile(r'^otpauth://(?P<type>\w+)'
                        r'/(?P<labe>[^?]+)'
                        r'\?(?P<query>.*)$')

LABEL   =   'label'
TYPE    =    'type'
ALGORITHM = 'algorithm'
DIGITS  =  'digits'
SECRET  =  'secret'
COUNTER = 'counter'
PERIOD  =  'period'
TOTP    =    'totp'
HOTP    =    'hotp'
DRIFT    =   'drift'

def parse_otpauth(otpauth_uri):
    m = re.match(otpauth_re, otpauth_uri)
    if not m:
        raise ValueError('Invalid otpauth URI', otpauth_uri)
    d = m.groupdict()
    query_parse = urlparse.parse_qs(d['query'])
    if SECRET not in query_parse:
        raise ValueError('Missing secret field in otpauth URI', otpauth_uri)
    try:
        d[SECRET] = base64.b32decode(query_parse[SECRET][0]).encode('hex')
    except TypeError:
        raise ValueError('Invalid base32 encoding of the secret field in '
                'otpauth URI', otpauth_uri)
    if ALGORITHM in query_parse:
        d[ALGORITHM] = query_parse[ALGORITHM].lower()
        if d[ALGORITHM] not in ('sha1', 'sha256', 'sha512', 'md5'):
            raise ValueError('Invalid value for algorithm field in otpauth '
                    'URI', otpauth_uri)
    else:
        d[ALGORITHM] = 'sha1'
    try:
        d[ALGORITHM] = getattr(hashlib, d[ALGORITHM])
    except AttributeError:
        raise ValueError('Unsupported algorithm %s in othauth URI' %
                d[ALGORITHM], otpauth_uri)
    for key in (DIGITS, PERIOD, COUNTER):
        try:
            if key in query_parse:
                d[key] = int(query_parse[key])
        except ValueError:
            raise ValueError('Invalid value for field %s in otpauth URI, must '
                    'be a number' % key, otpauth_uri)
    if COUNTER not in d:
        d[COUNTER] = 0 # what else ?
    if DIGITS in d:
        if d[DIGITS] not in (6,8):
            raise ValueError('Invalid value for field digits in othauth URI, it '
                    'must 6 or 8', otpauth_uri)
    else:
        d[DIGITS] = 6
    if d[TYPE] == HOTP and COUNTER not in d:
        raise ValueError('Missing field counter in otpauth URI, it is '
                'mandatory with the hotp type', otpauth_uri)
    if d[TYPE] == TOTP and PERIOD not in d:
        d[PERIOD] = 30
    return d

def from_b32key(b32_key, state=None):
    '''Some phone app directly accept a partial b32 encoding, we try to emulate that'''
    if len(b32_key) % 8 not in (0, 2, 4, 5, 7):
        raise ValueError('invalid base32 value')
    b32_key += '=' * (8 - len(b32_key) % 8)
    b32_key = b32_key.upper()
    try:
        base64.b32decode(b32_key)
    except TypeError:
        raise ValueError('invalid base32 value')
    return GoogleAuthenticator('otpauth://totp/xxx?%s' %
            urllib.urlencode({'secret': b32_key}), state=state)

class GoogleAuthenticator(object):
    def __init__(self, otpauth_uri, state=None):
        self.otpauth_uri = otpauth_uri
        self.parsed_otpauth_uri = parse_otpauth(otpauth_uri)
        self.generator_state = state or {}
        self.acceptor_state = state or {}

    def generate(self, t=None):
        format = 'dec%s' % self.parsed_otpauth_uri[DIGITS]
        hash = self.parsed_otpauth_uri[ALGORITHM]
        secret = self.parsed_otpauth_uri[SECRET]
        state = self.generator_state
        if self.parsed_otpauth_uri[TYPE] == HOTP:
            if COUNTER not in state:
                state[COUNTER] = self.parsed_otpauth_uri[COUNTER]
            otp = hotp.hotp(secret, state[COUNTER], format=format, hash=hash)
            state[COUNTER] += 1
            return otp
        elif self.parsed_otpauth_uri[TYPE] == TOTP:
            period = self.parsed_otpauth_uri[PERIOD]
            return totp.totp(secret, format=format, period=period, hash=hash, t=t)
        else:
            raise NotImplemented(self.parsed_otpauth_uri[TYPE])

    def accept(self, otp, hotp_drift=3, forward_drift=None,
            hotp_backward_drift=0, totp_forward_drift=1,
            totp_backward_drift=1, t=None):
        format = 'dec%s' % self.parsed_otpauth_uri[DIGITS]
        hash = self.parsed_otpauth_uri[ALGORITHM]
        secret = self.parsed_otpauth_uri[SECRET]
        state = self.acceptor_state
        if self.parsed_otpauth_uri[TYPE] == HOTP:
            if COUNTER not in state:
                state[COUNTER] = self.parsed_otpauth_uri[COUNTER]
            ok, state[COUNTER] = hotp.accept_hotp(otp, secret, state[COUNTER],
                    format=format, hash=hash, drift=hotp_drift,
                    backward_drift=hotp_backward_drift)
            return ok
        elif self.parsed_otpauth_uri[TYPE] == TOTP:
            period = 'dec%s' % self.parsed_otpauth_uri[PERIOD]
            if DRIFT not in state:
                state[DRIFT] = 0
            ok, state[DRIFT] = totp.accept_totp(secret, otp, format=format,
                    period=period, forward_drift=totp_forward_drift,
                    backward_drift=totp_backward_drift, drift=state[DRIFT],
                    t=t)
            return ok
        else:
            raise NotImplementedError(self.parsed_otpauth_uri[TYPE])
<commit_msg>google_authenticator: raise NotImplementedError not NotImplemented<commit_after>'''
Google Authenticator API
------------------------

Google Authenticator is based on HOTP and TOTP. It provides a simple way of
provisionning an OTP generator through a new URL scheme.

This module provides parsing and high-level API over the classic HOTP and TOTP
APIs provided by the oath.hotp and oath.totp modules.
'''

import re
import urlparse
import base64
import hashlib
import urllib

from . import _hotp as hotp
from . import _totp as totp

__all__ = ('GoogleAuthenticator', 'from_b32key')

otpauth_re = re.compile(r'^otpauth://(?P<type>\w+)'
                        r'/(?P<labe>[^?]+)'
                        r'\?(?P<query>.*)$')

LABEL   =   'label'
TYPE    =    'type'
ALGORITHM = 'algorithm'
DIGITS  =  'digits'
SECRET  =  'secret'
COUNTER = 'counter'
PERIOD  =  'period'
TOTP    =    'totp'
HOTP    =    'hotp'
DRIFT    =   'drift'

def parse_otpauth(otpauth_uri):
    m = re.match(otpauth_re, otpauth_uri)
    if not m:
        raise ValueError('Invalid otpauth URI', otpauth_uri)
    d = m.groupdict()
    query_parse = urlparse.parse_qs(d['query'])
    if SECRET not in query_parse:
        raise ValueError('Missing secret field in otpauth URI', otpauth_uri)
    try:
        d[SECRET] = base64.b32decode(query_parse[SECRET][0]).encode('hex')
    except TypeError:
        raise ValueError('Invalid base32 encoding of the secret field in '
                'otpauth URI', otpauth_uri)
    if ALGORITHM in query_parse:
        d[ALGORITHM] = query_parse[ALGORITHM].lower()
        if d[ALGORITHM] not in ('sha1', 'sha256', 'sha512', 'md5'):
            raise ValueError('Invalid value for algorithm field in otpauth '
                    'URI', otpauth_uri)
    else:
        d[ALGORITHM] = 'sha1'
    try:
        d[ALGORITHM] = getattr(hashlib, d[ALGORITHM])
    except AttributeError:
        raise ValueError('Unsupported algorithm %s in othauth URI' %
                d[ALGORITHM], otpauth_uri)
    for key in (DIGITS, PERIOD, COUNTER):
        try:
            if key in query_parse:
                d[key] = int(query_parse[key])
        except ValueError:
            raise ValueError('Invalid value for field %s in otpauth URI, must '
                    'be a number' % key, otpauth_uri)
    if COUNTER not in d:
        d[COUNTER] = 0 # what else ?
    if DIGITS in d:
        if d[DIGITS] not in (6,8):
            raise ValueError('Invalid value for field digits in othauth URI, it '
                    'must 6 or 8', otpauth_uri)
    else:
        d[DIGITS] = 6
    if d[TYPE] == HOTP and COUNTER not in d:
        raise ValueError('Missing field counter in otpauth URI, it is '
                'mandatory with the hotp type', otpauth_uri)
    if d[TYPE] == TOTP and PERIOD not in d:
        d[PERIOD] = 30
    return d

def from_b32key(b32_key, state=None):
    '''Some phone app directly accept a partial b32 encoding, we try to emulate that'''
    if len(b32_key) % 8 not in (0, 2, 4, 5, 7):
        raise ValueError('invalid base32 value')
    b32_key += '=' * (8 - len(b32_key) % 8)
    b32_key = b32_key.upper()
    try:
        base64.b32decode(b32_key)
    except TypeError:
        raise ValueError('invalid base32 value')
    return GoogleAuthenticator('otpauth://totp/xxx?%s' %
            urllib.urlencode({'secret': b32_key}), state=state)

class GoogleAuthenticator(object):
    def __init__(self, otpauth_uri, state=None):
        self.otpauth_uri = otpauth_uri
        self.parsed_otpauth_uri = parse_otpauth(otpauth_uri)
        self.generator_state = state or {}
        self.acceptor_state = state or {}

    def generate(self, t=None):
        format = 'dec%s' % self.parsed_otpauth_uri[DIGITS]
        hash = self.parsed_otpauth_uri[ALGORITHM]
        secret = self.parsed_otpauth_uri[SECRET]
        state = self.generator_state
        if self.parsed_otpauth_uri[TYPE] == HOTP:
            if COUNTER not in state:
                state[COUNTER] = self.parsed_otpauth_uri[COUNTER]
            otp = hotp.hotp(secret, state[COUNTER], format=format, hash=hash)
            state[COUNTER] += 1
            return otp
        elif self.parsed_otpauth_uri[TYPE] == TOTP:
            period = self.parsed_otpauth_uri[PERIOD]
            return totp.totp(secret, format=format, period=period, hash=hash, t=t)
        else:
            raise NotImplementedError(self.parsed_otpauth_uri[TYPE])

    def accept(self, otp, hotp_drift=3, forward_drift=None,
            hotp_backward_drift=0, totp_forward_drift=1,
            totp_backward_drift=1, t=None):
        format = 'dec%s' % self.parsed_otpauth_uri[DIGITS]
        hash = self.parsed_otpauth_uri[ALGORITHM]
        secret = self.parsed_otpauth_uri[SECRET]
        state = self.acceptor_state
        if self.parsed_otpauth_uri[TYPE] == HOTP:
            if COUNTER not in state:
                state[COUNTER] = self.parsed_otpauth_uri[COUNTER]
            ok, state[COUNTER] = hotp.accept_hotp(otp, secret, state[COUNTER],
                    format=format, hash=hash, drift=hotp_drift,
                    backward_drift=hotp_backward_drift)
            return ok
        elif self.parsed_otpauth_uri[TYPE] == TOTP:
            period = 'dec%s' % self.parsed_otpauth_uri[PERIOD]
            if DRIFT not in state:
                state[DRIFT] = 0
            ok, state[DRIFT] = totp.accept_totp(secret, otp, format=format,
                    period=period, forward_drift=totp_forward_drift,
                    backward_drift=totp_backward_drift, drift=state[DRIFT],
                    t=t)
            return ok
        else:
            raise NotImplementedError(self.parsed_otpauth_uri[TYPE])
<|endoftext|>