<commit_before># Copyright (C) 2020 OpenMotics BV
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import

import unittest
from mock import Mock
from ioc import SetTestMode, SetUpTestInjections
from gateway.dto import InputDTO
from gateway.hal.mappers_core import InputMapper
from master.core.core_communicator import CoreCommunicator
from master.core.basic_action import BasicAction
from master.core.memory_models import InputConfiguration
from master.core.memory_file import MemoryTypes, MemoryFile

class InputCoreMapperTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        SetTestMode()

    def setUp(self):
        self.memory = {}

        def _do_command(command, fields, timeout=None):
            _ = timeout
            instruction = ''.join(str(chr(c)) for c in command.instruction)
            if instruction == 'MR':
                page = fields['page']
                start = fields['start']
                length = fields['length']
                return {'data': self.memory.get(page, bytearray([255] * 256))[start:start + length]}
            elif instruction == 'MW':
                page = fields['page']
                start = fields['start']
                page_data = self.memory.setdefault(page, bytearray([255] * 256))
                for index, data_byte in enumerate(fields['data']):
                    page_data[start + index] = data_byte
            else:
                raise AssertionError('unexpected instruction: {0}'.format(instruction))

        self.communicator = Mock(CoreCommunicator)
        self.communicator.do_command = _do_command
        SetUpTestInjections(master_communicator=self.communicator)

        eeprom_file = MemoryFile(MemoryTypes.EEPROM)
        eeprom_file._cache = self.memory
        SetUpTestInjections(memory_files={MemoryTypes.EEPROM: eeprom_file,
                                          MemoryTypes.FRAM: MemoryFile(MemoryTypes.FRAM)})

    def test_mapping_basic(self):
        orm = InputMapper.dto_to_orm(InputDTO(id=0,
                                              name='input 0',
                                              event_enabled=True,
                                              invert=True),
                                     fields=['name', 'event_enabled', 'invert'])
        self.assertEqual(0, orm.id)
        self.assertEqual('input 0', orm.name)
        self.assertFalse(orm.input_config.normal_open)

        dto = InputMapper.orm_to_dto(InputConfiguration.deserialize({'id': 1,
                                                                     'name': 'input 1',
                                                                     'input_config': {'normal_open': True},
                                                                     'module': {'id': 0,
                                                                                'device_type': 'b'}}))
        self.assertEqual(1, dto.id)
        self.assertEqual('input 1', dto.name)
        self.assertEqual('I', dto.module_type)
        self.assertFalse(dto.invert)

    def test_actions_non_configured_input(self):
        orm = InputCoreMapperTest._dto_to_orm(action=255, basic_actions=[])
        self._validate_orm(orm,
                           enable_press_and_release=True,
                           enable_1s_press=True,
                           enable_2s_press=True,
                           enable_double_press=True)

        dto = InputCoreMapperTest._orm_to_dto()
        self.assertEqual(255, dto.action)
        self.assertEqual([], dto.basic_actions)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'output_id': 1023})
        self.assertEqual(255, dto.action)
        self.assertEqual([], dto.basic_actions)

    def test_actions_output_linked(self):
        orm = InputCoreMapperTest._dto_to_orm(action=123, basic_actions=[])
        self._validate_orm(orm,
                           has_direct_output_link=True,
                           in_use=True,
                           output_id=123)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'output_id': 123,
                                                          'enable_press_and_release': False,
                                                          'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False})
        self.assertEqual(123, dto.action)
        self.assertEqual([], dto.basic_actions)

    def test_actions_delayed_press(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[207, 1])
        self._validate_orm(orm,
                           in_use=True,
                           enable_2s_press=True,
                           basic_action_2s_press=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_2s_press)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_press_and_release': False,
                                                          'enable_1s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_2s_press=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([207, 1], dto.basic_actions)

    def test_actions_press(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[2, 1])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_press=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_press)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_press=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([2, 1], dto.basic_actions)

    def test_actions_release(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[236, 0, 2, 1, 236, 255])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_release=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_release)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_release=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([236, 0, 2, 1, 236, 255], dto.basic_actions)

    def test_actions_press_release(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[2, 1, 236, 0, 2, 2, 236, 255])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_press=True,
                           basic_action_release=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_press)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=2), orm.basic_action_release)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_press=BasicAction(action_type=19, action=0, device_nr=1),
                                              basic_action_release=BasicAction(action_type=19, action=0, device_nr=2))
        self.assertEqual(240, dto.action)
        self.assertEqual([2, 1, 236, 0, 2, 2, 236, 255], dto.basic_actions)

    def test_actions_invalid(self):
        with self.assertRaises(ValueError):
            InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[161, 1])

    def _validate_orm(self, orm, **kwargs):
        self.assertEqual(kwargs.get('has_direct_output_link', False), orm.has_direct_output_link)
        self.assertEqual(kwargs.get('in_use', False), orm.in_use)
        self.assertEqual(kwargs.get('output_id', 1023), orm.input_link.output_id)
        self.assertEqual(kwargs.get('enable_press_and_release', False), orm.input_link.enable_press_and_release)
        self.assertEqual(kwargs.get('enable_1s_press', False), orm.input_link.enable_1s_press)
        self.assertEqual(kwargs.get('enable_2s_press', False), orm.input_link.enable_2s_press)
        self.assertEqual(kwargs.get('enable_double_press', False), orm.input_link.enable_double_press)
        self.assertEqual(kwargs.get('basic_action_press', False), orm.basic_action_press.in_use)
        self.assertEqual(kwargs.get('basic_action_release', False), orm.basic_action_release.in_use)
        self.assertEqual(kwargs.get('basic_action_1s_press', False), orm.basic_action_1s_press.in_use)
        self.assertEqual(kwargs.get('basic_action_2s_press', False), orm.basic_action_2s_press.in_use)
        self.assertEqual(kwargs.get('basic_action_double_press', False), orm.basic_action_double_press.in_use)

    @staticmethod
    def _dto_to_orm(action, basic_actions):
        return InputMapper.dto_to_orm(InputDTO(id=0,
                                               name='input 0',
                                               action=action,
                                               basic_actions=basic_actions),
                                      fields=['name', 'action', 'basic_actions'])

    @staticmethod
    def _orm_to_dto(**kwargs):
        kwargs.update({'id': 0, 'name': 'input 0'})
        return InputMapper.orm_to_dto(InputConfiguration.deserialize(kwargs))
<commit_msg>Fixed test<commit_after># Copyright (C) 2020 OpenMotics BV
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import

import unittest
from mock import Mock
from ioc import SetTestMode, SetUpTestInjections
from gateway.dto import InputDTO
from gateway.hal.mappers_core import InputMapper
from master.core.core_communicator import CoreCommunicator
from master.core.basic_action import BasicAction
from master.core.memory_models import InputConfiguration, InputModuleConfiguration
from master.core.memory_file import MemoryTypes, MemoryFile

class InputCoreMapperTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        SetTestMode()

    def setUp(self):
        self.memory = {}

        def _do_command(command, fields, timeout=None):
            _ = timeout
            instruction = ''.join(str(chr(c)) for c in command.instruction)
            if instruction == 'MR':
                page = fields['page']
                start = fields['start']
                length = fields['length']
                return {'data': self.memory.get(page, bytearray([255] * 256))[start:start + length]}
            elif instruction == 'MW':
                page = fields['page']
                start = fields['start']
                page_data = self.memory.setdefault(page, bytearray([255] * 256))
                for index, data_byte in enumerate(fields['data']):
                    page_data[start + index] = data_byte
            else:
                raise AssertionError('unexpected instruction: {0}'.format(instruction))

        self.communicator = Mock(CoreCommunicator)
        self.communicator.do_command = _do_command
        SetUpTestInjections(master_communicator=self.communicator)

        eeprom_file = MemoryFile(MemoryTypes.EEPROM)
        eeprom_file._cache = self.memory
        SetUpTestInjections(memory_files={MemoryTypes.EEPROM: eeprom_file,
                                          MemoryTypes.FRAM: MemoryFile(MemoryTypes.FRAM)})

        # Remove read-only flags from device_type for testing purposes below
        if hasattr(InputModuleConfiguration, '_device_type'):
            InputModuleConfiguration._device_type._read_only = False
        else:
            InputModuleConfiguration.device_type._read_only = False

    def test_mapping_basic(self):
        orm = InputMapper.dto_to_orm(InputDTO(id=0,
                                              name='input 0',
                                              event_enabled=True,
                                              invert=True),
                                     fields=['name', 'event_enabled', 'invert'])
        self.assertEqual(0, orm.id)
        self.assertEqual('input 0', orm.name)
        self.assertFalse(orm.input_config.normal_open)

        dto = InputMapper.orm_to_dto(InputConfiguration.deserialize({'id': 1,
                                                                     'name': 'input 1',
                                                                     'input_config': {'normal_open': True},
                                                                     'module': {'id': 0,
                                                                                'device_type': 'b'}}))
        self.assertEqual(1, dto.id)
        self.assertEqual('input 1', dto.name)
        self.assertEqual('I', dto.module_type)
        self.assertFalse(dto.invert)

    def test_actions_non_configured_input(self):
        orm = InputCoreMapperTest._dto_to_orm(action=255, basic_actions=[])
        self._validate_orm(orm,
                           enable_press_and_release=True,
                           enable_1s_press=True,
                           enable_2s_press=True,
                           enable_double_press=True)

        dto = InputCoreMapperTest._orm_to_dto()
        self.assertEqual(255, dto.action)
        self.assertEqual([], dto.basic_actions)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'output_id': 1023})
        self.assertEqual(255, dto.action)
        self.assertEqual([], dto.basic_actions)

    def test_actions_output_linked(self):
        orm = InputCoreMapperTest._dto_to_orm(action=123, basic_actions=[])
        self._validate_orm(orm,
                           has_direct_output_link=True,
                           in_use=True,
                           output_id=123)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'output_id': 123,
                                                          'enable_press_and_release': False,
                                                          'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False})
        self.assertEqual(123, dto.action)
        self.assertEqual([], dto.basic_actions)

    def test_actions_delayed_press(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[207, 1])
        self._validate_orm(orm,
                           in_use=True,
                           enable_2s_press=True,
                           basic_action_2s_press=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_2s_press)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_press_and_release': False,
                                                          'enable_1s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_2s_press=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([207, 1], dto.basic_actions)

    def test_actions_press(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[2, 1])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_press=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_press)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_press=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([2, 1], dto.basic_actions)

    def test_actions_release(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[236, 0, 2, 1, 236, 255])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_release=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_release)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_release=BasicAction(action_type=19, action=0, device_nr=1))
        self.assertEqual(240, dto.action)
        self.assertEqual([236, 0, 2, 1, 236, 255], dto.basic_actions)

    def test_actions_press_release(self):
        orm = InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[2, 1, 236, 0, 2, 2, 236, 255])
        self._validate_orm(orm,
                           in_use=True,
                           enable_press_and_release=True,
                           basic_action_press=True,
                           basic_action_release=True)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=1), orm.basic_action_press)
        self.assertEqual(BasicAction(action_type=19, action=0, device_nr=2), orm.basic_action_release)

        dto = InputCoreMapperTest._orm_to_dto(input_link={'enable_1s_press': False,
                                                          'enable_2s_press': False,
                                                          'enable_double_press': False},
                                              basic_action_press=BasicAction(action_type=19, action=0, device_nr=1),
                                              basic_action_release=BasicAction(action_type=19, action=0, device_nr=2))
        self.assertEqual(240, dto.action)
        self.assertEqual([2, 1, 236, 0, 2, 2, 236, 255], dto.basic_actions)

    def test_actions_invalid(self):
        with self.assertRaises(ValueError):
            InputCoreMapperTest._dto_to_orm(action=240, basic_actions=[161, 1])

    def _validate_orm(self, orm, **kwargs):
        self.assertEqual(kwargs.get('has_direct_output_link', False), orm.has_direct_output_link)
        self.assertEqual(kwargs.get('in_use', False), orm.in_use)
        self.assertEqual(kwargs.get('output_id', 1023), orm.input_link.output_id)
        self.assertEqual(kwargs.get('enable_press_and_release', False), orm.input_link.enable_press_and_release)
        self.assertEqual(kwargs.get('enable_1s_press', False), orm.input_link.enable_1s_press)
        self.assertEqual(kwargs.get('enable_2s_press', False), orm.input_link.enable_2s_press)
        self.assertEqual(kwargs.get('enable_double_press', False), orm.input_link.enable_double_press)
        self.assertEqual(kwargs.get('basic_action_press', False), orm.basic_action_press.in_use)
        self.assertEqual(kwargs.get('basic_action_release', False), orm.basic_action_release.in_use)
        self.assertEqual(kwargs.get('basic_action_1s_press', False), orm.basic_action_1s_press.in_use)
        self.assertEqual(kwargs.get('basic_action_2s_press', False), orm.basic_action_2s_press.in_use)
        self.assertEqual(kwargs.get('basic_action_double_press', False), orm.basic_action_double_press.in_use)

    @staticmethod
    def _dto_to_orm(action, basic_actions):
        return InputMapper.dto_to_orm(InputDTO(id=0,
                                               name='input 0',
                                               action=action,
                                               basic_actions=basic_actions),
                                      fields=['name', 'action', 'basic_actions'])

    @staticmethod
    def _orm_to_dto(**kwargs):
        kwargs.update({'id': 0, 'name': 'input 0'})
        return InputMapper.orm_to_dto(InputConfiguration.deserialize(kwargs))
<|endoftext|>


<commit_before>from .utils import do_ex, trace
from .version import meta
from os.path import abspath, normcase, realpath

FILES_COMMAND = 'git ls-files'
DEFAULT_DESCRIBE = 'git describe --tags --long --match *.*'

def _normalized(path):
    return normcase(abspath(realpath(path)))

class GitWorkdir(object):
    def __init__(self, path):
        self.path = path

    def do_ex(self, cmd):
        return do_ex(cmd, cwd=self.path)

    @classmethod
    def from_potential_worktree(cls, wd):
        real_wd, _, ret = do_ex('git rev-parse --show-toplevel', wd)
        if ret:
            return
        trace('real root', real_wd)
        if _normalized(real_wd) != _normalized(wd):
            return

        return cls(real_wd)

    def is_dirty(self):
        out, _, _ = self.do_ex("git status --porcelain --untracked-files=no")
        return bool(out)

    def node(self):
        rev_node, _, ret = self.do_ex('git rev-parse --verify --quiet HEAD')
        if not ret:
            return rev_node[:7]

    def count_all_nodes(self):
        revs, _, _ = self.do_ex('git rev-list HEAD')
        return revs.count('\n') + 1

def parse(root, describe_command=DEFAULT_DESCRIBE):
    wd = GitWorkdir(root)

    rev_node = wd.node()
    dirty = wd.is_dirty()

    if rev_node is None:
        return meta('0.0', dirty=dirty)

    out, err, ret = do_ex(describe_command, root)
    if ret:
        return meta(
            '0.0',
            distance=wd.count_all_nodes(),
            node=rev_node,
            dirty=dirty,
        )

    tag, number, node = out.rsplit('-', 2)
    number = int(number)
    if number:
        return meta(tag, distance=number, node=node, dirty=dirty)
    else:
        return meta(tag, node=node, dirty=dirty)
<commit_msg>fix oversight - when git node is None, distance should be 0<commit_after>from .utils import do_ex, trace
from .version import meta
from os.path import abspath, normcase, realpath

FILES_COMMAND = 'git ls-files'
DEFAULT_DESCRIBE = 'git describe --tags --long --match *.*'

def _normalized(path):
    return normcase(abspath(realpath(path)))

class GitWorkdir(object):
    def __init__(self, path):
        self.path = path

    def do_ex(self, cmd):
        return do_ex(cmd, cwd=self.path)

    @classmethod
    def from_potential_worktree(cls, wd):
        real_wd, _, ret = do_ex('git rev-parse --show-toplevel', wd)
        if ret:
            return
        trace('real root', real_wd)
        if _normalized(real_wd) != _normalized(wd):
            return

        return cls(real_wd)

    def is_dirty(self):
        out, _, _ = self.do_ex("git status --porcelain --untracked-files=no")
        return bool(out)

    def node(self):
        rev_node, _, ret = self.do_ex('git rev-parse --verify --quiet HEAD')
        if not ret:
            return rev_node[:7]

    def count_all_nodes(self):
        revs, _, _ = self.do_ex('git rev-list HEAD')
        return revs.count('\n') + 1

def parse(root, describe_command=DEFAULT_DESCRIBE):
    wd = GitWorkdir(root)

    rev_node = wd.node()
    dirty = wd.is_dirty()

    if rev_node is None:
        return meta('0.0', distance=0, dirty=dirty)

    out, err, ret = do_ex(describe_command, root)
    if ret:
        return meta(
            '0.0',
            distance=wd.count_all_nodes(),
            node=rev_node,
            dirty=dirty,
        )

    tag, number, node = out.rsplit('-', 2)
    number = int(number)
    if number:
        return meta(tag, distance=number, node=node, dirty=dirty)
    else:
        return meta(tag, node=node, dirty=dirty)
<|endoftext|>


<commit_before>"""
cclib (http://cclib.sf.net) is (c) 2006, the cclib development team
and licensed under the LGPL (http://www.gnu.org/copyleft/lgpl.html).
"""

__revision__ = "$Revision: 635 $"

import re

# If numpy is not installed, try to import Numeric instead.
try:
    import numpy
except ImportError:
    import Numeric as numpy

import utils
import logfileparser

class AtomBasis:
    def __init__(self, atname, basis_name, inputfile):
        self.symmetries=[]
        self.coefficients=[]
        self.atname=atname
        self.basis_name=basis_name

        self.parse_basis(inputfile)

    def parse_basis(self, inputfile):
        i=0
        line=inputfile.next()

        while(line[0]!="*"):
            (nbasis_text, symm)=line.split()
            self.symmetries.append(symm)

            nbasis=int(nbasis_text)
            coeff_arr=numpy.zeros((nbasis, 2), float)

            for j in range(0, nbasis, 1):
                line=inputfile.next()
                (e1_text, e2_text)=line.split()
                coeff_arr[j][0]=float(e1_text)
                coeff_arr[j][1]=float(e2_text)

            self.coefficients.append(coeff_arr)
            line=inputfile.next()

class Turbomole(logfileparser.Logfile):
    """A Turbomole output file"""

    def __init__(self, *args):

        # Call the __init__ method of the superclass
        super(Turbomole, self).__init__(logname="Turbomole", *args)

    def __str__(self):
        """Return a string representation of the object."""
        return "Turbomole output file %s" % (self.filename)

    def __repr__(self):
        """Return a representation of the object."""
        return 'Turbomole("%s")' % (self.filename)

    def atlist(self, atstr):
        # turn atstr from atoms section into array

        fields=atstr.split(',')
        list=[]
        for f in fields:
            if(f.find('-')!=-1):
                rangefields=f.split('-')
                start=int(rangefields[0])
                end=int(rangefields[1])

                for j in range(start, end+1, 1):
                    list.append(j-1)
            else:
                list.append(int(f)-1)
        return(list)

    def normalisesym(self, label):
        """Normalise the symmetries used by Turbomole."""
        return ans

    def before_parsing(self):
        self.geoopt = False # Is this a GeoOpt? Needed for SCF targets/values.

    def split_molines(self, inline):
        line=inline.replace("D", "E")
        f1=line[0:20]
        f2=line[20:40]
        f3=line[40:60]
        f4=line[60:80]

        if(len(f4)>1):
            return( (float(f1), float(f2), float(f3), float(f4)) )
        if(len(f3)>1):
            return( (float(f1), float(f2), float(f3)) )
        if(len(f2)>1):
            return( (float(f1), float(f2)) )
        if(len(f1)>1):
            return([float(f1)])
        return

    def extract(self, inputfile, line):
        """Extract information from the file object inputfile."""

        if line[3:11]=="nbf(AO)=":
            nmo=int(line[11:])
            self.nbasis=nmo
            self.nmo=nmo
        if line[3:9]=="nshell":
            temp=line.split('=')
            homos=int(temp[1])

        if line == "$basis\n":
            self.basis_lib=[]
            line = inputfile.next()
            line = inputfile.next()

            while line[0] != '*' and line[0] != '$':
                temp=line.split()
                line = inputfile.next()
                while line[0]=="#":
                    line = inputfile.next()
                self.basis_lib.append(AtomBasis(temp[0], temp[1], inputfile))
                line = inputfile.next()

        if line == "$coord\n":
            self.atomcoords = []
            self.atomnos = []
            atomcoords = []
            atomnos = []

            line = inputfile.next()

            while line[0] != "$":
                temp = line.split()
                atsym=temp[3].capitalize()
                atomnos.append(self.table.number[atsym])
                atomcoords.append([utils.convertor(float(x), "bohr", "Angstrom")
                                   for x in temp[0:3]])
                line = inputfile.next()
            self.atomcoords.append(atomcoords)
            self.atomnos = numpy.array(atomnos, "i")

        if line == "$atoms\n":
            line = inputfile.next()
            self.atomlist=[]
            while line[0]!="$":
                temp=line.split()
                at=temp[0]
                atnosstr=temp[1]
                while temp[1][len(temp[1])-1] == ',':
                    line = inputfile.next()
                    temp=line.split()
                    atnosstr=atnosstr+temp[0]
                atlist=self.atlist(atnosstr)

                line = inputfile.next()

                temp=line.split()
                basisname=temp[2]

                line = inputfile.next()
                while(line.find('jbas')!=-1 or line.find('ecp')!=-1 or
                      line.find('jkbas')!=-1):
                    line = inputfile.next()

                self.atomlist.append( (at, basisname, atlist))

        if line[3:10]=="natoms=":
            self.natom=int(line[10:])
            basistable=[]

            self.atomlist
            for i in range(0, self.natom, 1):
                for j in range(0, len(self.atomlist), 1):
                    for k in range(0, len(self.atomlist[j][2]), 1):
                        if(self.atomlist[j][2][k]==i):
                            basistable.append((self.atomlist[j][0],
                                                   self.atomlist[j][1]))
            self.aonames=[]
            counter=1
            for a, b in basistable:
                for i in range(0, len(self.basis_lib), 1):
                    if self.basis_lib[i].atname==a and self.basis_lib[i].basis_name==b:
                        pa=a.capitalize()

                        basis=self.basis_lib[i]

                        s_counter=1
                        p_counter=2
                        d_counter=3
                        f_counter=4
                        g_counter=5

                        for j in range(0, len(basis.symmetries), 1):
                            if basis.symmetries[j]=='s':
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, s_counter, "S"))
                                s_counter=s_counter+1
                            elif basis.symmetries[j]=='p':
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PX"))
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PY"))
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PZ"))
                                p_counter=p_counter+1
                            elif basis.symmetries[j]=='d':
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D 0"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D+1"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D-1"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D+2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D-2"))
                                d_counter=d_counter+1
                            elif basis.symmetries[j]=='f':
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F 0"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+1"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F-1"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+2"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F-2"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+3"))
                                 self.aonames.append("%s%d_%d%s" % \
                                        (pa, counter, f_counter, "F-3"))
                            elif basis.symmetries[j]=='g':
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G 0"))
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G+1"))
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G-1"))
                                self.aonames.append("%s%d_%d%s" % \
                                        (pa, counter, g_counter, "G+2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, g_counter, "G-2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, g_counter, "G+3"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G-3"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G+4"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G-4"))
                        break
                counter=counter+1

        if line=="$closed shells\n":
            line = inputfile.next()
            temp = line.split()
            occs = int(temp[1][2:])
            self.homos = numpy.array([occs-1], "i")
        if line[12:24]=="OVERLAP(CAO)":
            line = inputfile.next()
            line = inputfile.next()
            overlaparray=[]
            self.aooverlaps=numpy.zeros( (self.nbasis, self.nbasis), "d")
            while line != "       ----------------------\n":
                temp=line.split()
                overlaparray.extend(map(float, temp))
                line = inputfile.next()
            counter=0

            for i in range(0, self.nbasis, 1):
                for j in range(0, i+1, 1):
                    self.aooverlaps[i][j]=overlaparray[counter]
                    self.aooverlaps[j][i]=overlaparray[counter]
                    counter=counter+1

        if line[0:6] == "$scfmo" and line.find("scfconv")>-1:
            self.moenergies=[]
            self.mocoeffs=[]
            moarray=[]
            title = inputfile.next()
            while(title[0] == "#"):
                title = inputfile.next()
            mocoeffs = numpy.zeros((self.nbasis, self.nbasis), "d")

            while(title[0] != '$'):
                temp=title.split()

                orb_symm=temp[1]
                orb_en=float(temp[2][20:].replace("D", "E"))

                self.moenergies.append(orb_en)
                single_mo = []

                while(len(single_mo)<self.nbasis):
                    title = inputfile.next()
                    lines_coeffs=self.split_molines(title)
                    single_mo.extend(lines_coeffs)

                moarray.append(single_mo)
                title = inputfile.next()

            for i in range(0, len(moarray), 1):
                for j in range(0, self.nbasis, 1):
                    mocoeffs[i][j]=moarray[i][j]

            self.mocoeffs.append(mocoeffs)
<commit_msg>Added Turbomole vibrational frequency parsing<commit_after>"""
cclib (http://cclib.sf.net) is (c) 2006, the cclib development team
and licensed under the LGPL (http://www.gnu.org/copyleft/lgpl.html).
"""

__revision__ = "$Revision: 635 $"

import re

# If numpy is not installed, try to import Numeric instead.
try:
    import numpy
except ImportError:
    import Numeric as numpy

import utils
import logfileparser

class AtomBasis:
    def __init__(self, atname, basis_name, inputfile):
        self.symmetries=[]
        self.coefficients=[]
        self.atname=atname
        self.basis_name=basis_name

        self.parse_basis(inputfile)

    def parse_basis(self, inputfile):
        i=0
        line=inputfile.next()

        while(line[0]!="*"):
            (nbasis_text, symm)=line.split()
            self.symmetries.append(symm)

            nbasis=int(nbasis_text)
            coeff_arr=numpy.zeros((nbasis, 2), float)

            for j in range(0, nbasis, 1):
                line=inputfile.next()
                (e1_text, e2_text)=line.split()
                coeff_arr[j][0]=float(e1_text)
                coeff_arr[j][1]=float(e2_text)

            self.coefficients.append(coeff_arr)
            line=inputfile.next()

class Turbomole(logfileparser.Logfile):
    """A Turbomole output file"""

    def __init__(self, *args):

        # Call the __init__ method of the superclass
        super(Turbomole, self).__init__(logname="Turbomole", *args)

    def __str__(self):
        """Return a string representation of the object."""
        return "Turbomole output file %s" % (self.filename)

    def __repr__(self):
        """Return a representation of the object."""
        return 'Turbomole("%s")' % (self.filename)

    def atlist(self, atstr):
        # turn atstr from atoms section into array

        fields=atstr.split(',')
        list=[]
        for f in fields:
            if(f.find('-')!=-1):
                rangefields=f.split('-')
                start=int(rangefields[0])
                end=int(rangefields[1])

                for j in range(start, end+1, 1):
                    list.append(j-1)
            else:
                list.append(int(f)-1)
        return(list)

    def normalisesym(self, label):
        """Normalise the symmetries used by Turbomole."""
        return ans

    def before_parsing(self):
        self.geoopt = False # Is this a GeoOpt? Needed for SCF targets/values.

    def split_molines(self, inline):
        line=inline.replace("D", "E")
        f1=line[0:20]
        f2=line[20:40]
        f3=line[40:60]
        f4=line[60:80]

        if(len(f4)>1):
            return( (float(f1), float(f2), float(f3), float(f4)) )
        if(len(f3)>1):
            return( (float(f1), float(f2), float(f3)) )
        if(len(f2)>1):
            return( (float(f1), float(f2)) )
        if(len(f1)>1):
            return([float(f1)])
        return

    def extract(self, inputfile, line):
        """Extract information from the file object inputfile."""

        if line[3:11]=="nbf(AO)=":
            nmo=int(line[11:])
            self.nbasis=nmo
            self.nmo=nmo
        if line[3:9]=="nshell":
            temp=line.split('=')
            homos=int(temp[1])

        if line == "$basis\n":
            self.basis_lib=[]
            line = inputfile.next()
            line = inputfile.next()

            while line[0] != '*' and line[0] != '$':
                temp=line.split()
                line = inputfile.next()
                while line[0]=="#":
                    line = inputfile.next()
                self.basis_lib.append(AtomBasis(temp[0], temp[1], inputfile))
                line = inputfile.next()
        if line == "$ecp\n":
            self.ecp_lib=[]

            line = inputfile.next()
            line = inputfile.next()

            while line[0] != '*' and line[0] != '$':
                fields=line.split()
                atname=fields[0]
                ecpname=fields[1]
                line = inputfile.next()
                line = inputfile.next()
                fields=line.split()
                ncore = int(fields[2])

                while line[0] != '*':
                    line = inputfile.next()
                self.ecp_lib.append([atname, ecpname, ncore])

        if line == "$coord\n":
            self.atomcoords = []
            self.atomnos = []
            atomcoords = []
            atomnos = []

            line = inputfile.next()

            while line[0] != "$":
                temp = line.split()
                atsym=temp[3].capitalize()
                atomnos.append(self.table.number[atsym])
                atomcoords.append([utils.convertor(float(x), "bohr", "Angstrom")
                                   for x in temp[0:3]])
                line = inputfile.next()
            self.atomcoords.append(atomcoords)
            self.atomnos = numpy.array(atomnos, "i")

        if line == "$atoms\n":
            line = inputfile.next()
            self.atomlist=[]
            while line[0]!="$":
                temp=line.split()
                at=temp[0]
                atnosstr=temp[1]
                while temp[1][len(temp[1])-1] == ',':
                    line = inputfile.next()
                    temp=line.split()
                    atnosstr=atnosstr+temp[0]
                atlist=self.atlist(atnosstr)

                line = inputfile.next()

                temp=line.split()
                basisname=temp[2]
                ecpname=''
                line = inputfile.next()
                while(line.find('jbas')!=-1 or line.find('ecp')!=-1 or
                      line.find('jkbas')!=-1):
                    if line.find('ecp')!=-1:
                        temp=line.split()
                        ecpname=temp[2]
                    line = inputfile.next()

                self.atomlist.append( (at, basisname, ecpname, atlist))

        if line[3:10]=="natoms=":
            self.natom=int(line[10:])
            basistable=[]

            for i in range(0, self.natom, 1):
                for j in range(0, len(self.atomlist), 1):
                    for k in range(0, len(self.atomlist[j][3]), 1):
                        if self.atomlist[j][3][k]==i:
                            basistable.append((self.atomlist[j][0],
                                                   self.atomlist[j][1],
                                               self.atomlist[j][2]))
            self.aonames=[]
            counter=1
            for a, b, c in basistable:
                ncore=0
                if len(c) > 0:
                    for i in range(0, len(self.ecp_lib), 1):
                        if self.ecp_lib[i][0]==a and \
                           self.ecp_lib[i][1]==c:
                            ncore=self.ecp_lib[i][2]

                for i in range(0, len(self.basis_lib), 1):
                    if self.basis_lib[i].atname==a and self.basis_lib[i].basis_name==b:
                        pa=a.capitalize()
                        basis=self.basis_lib[i]

                        s_counter=1
                        p_counter=2
                        d_counter=3
                        f_counter=4
                        g_counter=5
# this is a really ugly piece of code to assign the right labels to
# basis functions on atoms with an ecp
                        if ncore == 2:
                            s_counter=2
                        elif ncore == 10:
                            s_counter=3
                            p_counter=3
                        elif ncore == 18:
                            s_counter=4
                            p_counter=4
                        elif ncore == 28:
                            s_counter=4
                            p_counter=4
                            d_counter=4
                        elif ncore == 36:
                            s_counter=5
                            p_counter=5
                            d_counter=5
                        elif ncore == 46:
                            s_counter=5
                            p_counter=5
                            d_counter=6

                        for j in range(0, len(basis.symmetries), 1):
                            if basis.symmetries[j]=='s':
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, s_counter, "S"))
                                s_counter=s_counter+1
                            elif basis.symmetries[j]=='p':
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PX"))
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PY"))
                                self.aonames.append("%s%d_%d%s" % \
                                              (pa, counter, p_counter, "PZ"))
                                p_counter=p_counter+1
                            elif basis.symmetries[j]=='d':
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D 0"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D+1"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D-1"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D+2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, d_counter, "D-2"))
                                d_counter=d_counter+1
                            elif basis.symmetries[j]=='f':
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F 0"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+1"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F-1"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+2"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F-2"))
                                 self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "F+3"))
                                 self.aonames.append("%s%d_%d%s" % \
                                        (pa, counter, f_counter, "F-3"))
                            elif basis.symmetries[j]=='g':
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G 0"))
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G+1"))
                                self.aonames.append("%s%d_%d%s" % \
                                       (pa, counter, f_counter, "G-1"))
                                self.aonames.append("%s%d_%d%s" % \
                                        (pa, counter, g_counter, "G+2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, g_counter, "G-2"))
                                self.aonames.append("%s%d_%d%s" % \
                                         (pa, counter, g_counter, "G+3"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G-3"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G+4"))
                                self.aonames.append("%s%d_%d%s" % \
                                          (pa, counter, g_counter, "G-4"))
                        break
                counter=counter+1

        if line=="$closed shells\n":
            line = inputfile.next()
            temp = line.split()
            occs = int(temp[1][2:])
            self.homos = numpy.array([occs-1], "i")
        if line[12:24]=="OVERLAP(CAO)":
            line = inputfile.next()
            line = inputfile.next()
            overlaparray=[]
            self.aooverlaps=numpy.zeros( (self.nbasis, self.nbasis), "d")
            while line != "       ----------------------\n":
                temp=line.split()
                overlaparray.extend(map(float, temp))
                line = inputfile.next()
            counter=0

            for i in range(0, self.nbasis, 1):
                for j in range(0, i+1, 1):
                    self.aooverlaps[i][j]=overlaparray[counter]
                    self.aooverlaps[j][i]=overlaparray[counter]
                    counter=counter+1

        if line[0:6] == "$scfmo" and line.find("scfconv")>-1:
            self.moenergies=[]
            self.mocoeffs=[]
            moarray=[]
            title = inputfile.next()
            while(title[0] == "#"):
                title = inputfile.next()
            mocoeffs = numpy.zeros((self.nbasis, self.nbasis), "d")

            while(title[0] != '$'):
                temp=title.split()

                orb_symm=temp[1]
                orb_en=float(temp[2][20:].replace("D", "E"))

                self.moenergies.append(orb_en)
                single_mo = []

                while(len(single_mo)<self.nbasis):
                    title = inputfile.next()
                    lines_coeffs=self.split_molines(title)
                    single_mo.extend(lines_coeffs)

                moarray.append(single_mo)
                title = inputfile.next()

            for i in range(0, len(moarray), 1):
                for j in range(0, self.nbasis, 1):
                    mocoeffs[i][j]=moarray[i][j]

            self.mocoeffs.append(mocoeffs)

        if line[26:49] == "a o f o r c e - program":
            self.vibirs = []
            self.vibfreqs = []
            self.vibsyms = []
            self.vibdisps = []

            while line[3:31] != "****  force : all done  ****":
                if line[5:14] == "frequency":
                    temp=line.split()

                    freqs = [self.float(f) for f in temp[1:]]
                    self.vibfreqs.extend(freqs)

                    line=inputfile.next()
                    line=inputfile.next()

                    syms=line.split()
                    self.vibsyms.extend(syms[1:])

                    line=inputfile.next()
                    line=inputfile.next()
                    line=inputfile.next()
                    line=inputfile.next()

                    temp=line.split()
                    irs = [self.float(f) for f in temp[2:]]
                    self.vibirs.extend(irs)

                    line=inputfile.next()
                    line=inputfile.next()
                    line=inputfile.next()
                    line=inputfile.next()

                    x=[]
                    y=[]
                    z=[]

                    line=inputfile.next()
                    while len(line) > 1:
                        temp=line.split()
                        x.append(map(float, temp[3:]))

                        line=inputfile.next()
                        temp=line.split()
                        y.append(map(float, temp[1:]))

                        line=inputfile.next()
                        temp=line.split()
                        z.append(map(float, temp[1:]))
                        line=inputfile.next()

                    for i in range(0, len(x[0]), 1):
                        disp=[]
                        for j in range(0, len(x), 1):
                            disp.append( [x[j][i], y[j][i], z[j][i]])
                        self.vibdisps.append(disp)

                line=inputfile.next()

            i=0
            while i<len(self.vibfreqs):
                if self.vibfreqs[i]==0.0:
                    del self.vibfreqs[i]
                    del self.vibdisps[i]
                    del self.vibirs[i]
                    del self.vibsyms[i]
                else:
                    i=i+1
                print str(i) + ' ' + str(self.vibfreqs)
<|endoftext|>


<commit_before>import chainer
from chainer.backends import cuda
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
import numpy as np
import unittest

from chainercv import functions

@testing.parameterize(*testing.product({
    'sampling_ratio': [None, 1, 2, (None, 3), (1, 2)],
}))
class TestPSROIAveragePolling2D(unittest.TestCase):

    def setUp(self):
        self.N = 3
        self.group_size = 2
        self.out_c = 2
        self.n_channels = self.group_size * self.group_size * self.out_c
        self.x = np.arange(
            self.N * self.n_channels * 10 * 12,
            dtype=np.float32).reshape((self.N, self.n_channels, 10, 12))
        np.random.shuffle(self.x)
        self.x = 2 * self.x / self.x.size - 1
        self.x = self.x.astype(np.float32)
        self.rois = np.array(
            [[0, 0, 7, 7],
             [1, 0, 5, 12],
             [0, 1, 10, 5],
             [3, 3, 4, 4]],
            dtype=np.float32
        )
        self.roi_indices = np.array([0, 2, 1, 0], dtype=np.int32)
        self.n_roi = self.rois.shape[0]
        self.out_h, self.out_w = 4, 4
        self.spatial_scale = 1.0
        self.gy = np.random.uniform(
            -1, 1, (self.n_roi, self.out_c, self.out_h, self.out_w))
        self.gy = self.gy.astype(np.float32)
        self.check_backward_options = {'atol': 5e-4, 'rtol': 5e-3}

    def check_forward(self, x_data, roi_data, roi_index_data):
        x = chainer.Variable(x_data)
        rois = chainer.Variable(roi_data)
        roi_indices = chainer.Variable(roi_index_data)
        y = functions.ps_roi_average_align_2d(
            x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
            self.spatial_scale, self.group_size,
            sampling_ratio=self.sampling_ratio)
        self.assertEqual(y.data.dtype, np.float32)
        y_data = cuda.to_cpu(y.data)
        self.assertEqual(
            (self.n_roi, self.out_c, self.out_h, self.out_w), y_data.shape)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x, self.rois, self.roi_indices)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices))

    def check_backward(self, x_data, roi_data, roi_index_data, y_grad_data):
        def f(x, rois, roi_indices):
            return functions.ps_roi_average_align_2d(
                x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
                self.spatial_scale, self.group_size,
                sampling_ratio=self.sampling_ratio)
        gradient_check.check_backward(
            f, (x_data, roi_data, roi_index_data), y_grad_data,
            no_grads=[False, True, True], **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.rois, self.roi_indices, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices), cuda.to_gpu(self.gy))

    def apply_backward(self, x_data, roi_data, roi_index_data, y_grad_data):
        x = chainer.Variable(x_data)
        rois = chainer.Variable(roi_data)
        roi_indices = chainer.Variable(roi_index_data)
        y = functions.ps_roi_average_align_2d(
            x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
            self.spatial_scale, self.group_size,
            sampling_ratio=self.sampling_ratio)
        x.cleargrad()
        y.grad = y_grad_data
        y.backward()
        return x, y

    @attr.gpu
    @condition.retry(3)
    def test_consistency_with_gpu(self):
        x_cpu, y_cpu = self.apply_backward(
            self.x, self.rois, self.roi_indices, self.gy)
        x_gpu, y_gpu = self.apply_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices), cuda.to_gpu(self.gy))
        testing.assert_allclose(y_cpu.data, y_gpu.data)
        testing.assert_allclose(x_cpu.grad, x_gpu.grad)

testing.run_module(__name__, __file__)
<commit_msg>fix typo in functions_tests<commit_after>import chainer
from chainer.backends import cuda
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
import numpy as np
import unittest

from chainercv import functions

@testing.parameterize(*testing.product({
    'sampling_ratio': [None, 1, 2, (None, 3), (1, 2)],
}))
class TestPSROIAverageAlign2D(unittest.TestCase):

    def setUp(self):
        self.N = 3
        self.group_size = 2
        self.out_c = 2
        self.n_channels = self.group_size * self.group_size * self.out_c
        self.x = np.arange(
            self.N * self.n_channels * 10 * 12,
            dtype=np.float32).reshape((self.N, self.n_channels, 10, 12))
        np.random.shuffle(self.x)
        self.x = 2 * self.x / self.x.size - 1
        self.x = self.x.astype(np.float32)
        self.rois = np.array(
            [[0, 0, 7, 7],
             [1, 0, 5, 12],
             [0, 1, 10, 5],
             [3, 3, 4, 4]],
            dtype=np.float32
        )
        self.roi_indices = np.array([0, 2, 1, 0], dtype=np.int32)
        self.n_roi = self.rois.shape[0]
        self.out_h, self.out_w = 4, 4
        self.spatial_scale = 1.0
        self.gy = np.random.uniform(
            -1, 1, (self.n_roi, self.out_c, self.out_h, self.out_w))
        self.gy = self.gy.astype(np.float32)
        self.check_backward_options = {'atol': 5e-4, 'rtol': 5e-3}

    def check_forward(self, x_data, roi_data, roi_index_data):
        x = chainer.Variable(x_data)
        rois = chainer.Variable(roi_data)
        roi_indices = chainer.Variable(roi_index_data)
        y = functions.ps_roi_average_align_2d(
            x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
            self.spatial_scale, self.group_size,
            sampling_ratio=self.sampling_ratio)
        self.assertEqual(y.data.dtype, np.float32)
        y_data = cuda.to_cpu(y.data)
        self.assertEqual(
            (self.n_roi, self.out_c, self.out_h, self.out_w), y_data.shape)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x, self.rois, self.roi_indices)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices))

    def check_backward(self, x_data, roi_data, roi_index_data, y_grad_data):
        def f(x, rois, roi_indices):
            return functions.ps_roi_average_align_2d(
                x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
                self.spatial_scale, self.group_size,
                sampling_ratio=self.sampling_ratio)
        gradient_check.check_backward(
            f, (x_data, roi_data, roi_index_data), y_grad_data,
            no_grads=[False, True, True], **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.rois, self.roi_indices, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices), cuda.to_gpu(self.gy))

    def apply_backward(self, x_data, roi_data, roi_index_data, y_grad_data):
        x = chainer.Variable(x_data)
        rois = chainer.Variable(roi_data)
        roi_indices = chainer.Variable(roi_index_data)
        y = functions.ps_roi_average_align_2d(
            x, rois, roi_indices, self.out_c, self.out_h, self.out_w,
            self.spatial_scale, self.group_size,
            sampling_ratio=self.sampling_ratio)
        x.cleargrad()
        y.grad = y_grad_data
        y.backward()
        return x, y

    @attr.gpu
    @condition.retry(3)
    def test_consistency_with_gpu(self):
        x_cpu, y_cpu = self.apply_backward(
            self.x, self.rois, self.roi_indices, self.gy)
        x_gpu, y_gpu = self.apply_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.rois),
            cuda.to_gpu(self.roi_indices), cuda.to_gpu(self.gy))
        testing.assert_allclose(y_cpu.data, y_gpu.data)
        testing.assert_allclose(x_cpu.grad, x_gpu.grad)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import numpy as nm
import os
import os.path as op
import fnmatch
import shutil
from base import output, Struct
try:
    import tables as pt
except:
    pt = None

class InDir(Struct):
    """
    Store the directory name a file is in, and prepend this name to other
    files.

    Examples
    --------

    >>> indir = InDir('output/file1')
    >>> print indir('file2')
    """
    def __init__(self, filename):
        self.dir = op.split(op.join(os.getcwd(), filename))[0]

    def __call__(self, filename):
        return op.join(self.dir, filename)

def ensure_path(filename):
    """
    Check if path to `filename` exists and if not, create the necessary
    intermediate directories.
    """
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

def locate_files(pattern, root_dir=os.curdir):
    """
    Locate all files matching fiven filename pattern in and below
    supplied root directory.
    """
    for dirpath, dirnames, filenames in os.walk(os.path.abspath(root_dir)):
        for filename in fnmatch.filter(filenames, pattern):
            yield os.path.join(dirpath, filename)

def remove_files(root_dir):
    """
    Remove all files and directories in supplied root directory.
    """
    for dirpath, dirnames, filenames in os.walk(os.path.abspath(root_dir)):
        for filename in filenames:
            os.remove(os.path.join(root_dir, filename))

        for dirname in dirnames:
            shutil.rmtree(os.path.join(root_dir, dirname))

##
# 27.04.2006, c
def get_trunk(filename):
    return op.splitext(op.basename(filename))[0]

def get_print_info(n_step, fill=None):
    """
    Returns the max. number of digits in range(n_step) and the corresponding
    format string.

    Examples:

    >>> get_print_info(11)
    (2, '%2d')
    >>> get_print_info(8)
    (1, '%1d')
    >>> get_print_info(100)
    (2, '%2d')
    >>> get_print_info(101)
    (3, '%3d')
    >>> get_print_info(101, fill='0')
    (3, '%03d')
    """
    if n_step > 1:
        n_digit = int(nm.log10(n_step - 1) + 1)
        if fill is None:
            format = '%%%dd' % n_digit
        else:
            format = '%%%s%dd' % (fill, n_digit)
    else:
        n_digit, format = 0, None
    return n_digit, format

def skip_read_line(fd, no_eof=False):
    """
    Read the first non-empty line (if any) from the given file
    object. Return an empty string at EOF, if `no_eof` is False. If it
    is True, raise the EOFError instead.
    """
    ls = ''
    while 1:
        try:
            line = fd.readline()

        except EOFError:
            break

        if not line:
            if no_eof:
                raise EOFError

            else:
                break

        ls = line.strip()
        if ls and (ls[0] != '#'):
            break

    return ls

def read_token(fd):
    """
    Read a single token (sequence of non-whitespace characters) from the
    given file object.

    Notes
    -----
    Consumes the first whitespace character after the token.
    """
    out = ''
    # Skip initial whitespace.

    while 1:
        ch = fd.read(1)
        if ch.isspace(): continue
        elif len(ch) == 0: return out
        else: break

    while not ch.isspace():
        out = out + ch
        ch = fd.read(1)
        if len(ch) == 0: break

    return out

def read_array(fd, n_row, n_col, dtype):
    """
    Read a NumPy array of shape `(n_row, n_col)` from the given file
    object and cast it to type `dtype`.
    """
    count = n_row * n_col
    val = nm.fromfile(fd, sep=' ', count=count)

    if val.shape[0] < count:
        raise ValueError('(%d, %d) array reading failed!' % (n_row, n_col))

    val = nm.asarray(val, dtype=dtype)
    val.shape = (n_row, n_col)

    return val

##
# c: 05.02.2008, r: 05.02.2008
def read_list(fd, n_item, dtype):
    vals = []
    ii = 0
    while ii < n_item:
        line = [dtype(ic) for ic in fd.readline().split()]
        vals.append(line)
        ii += len(line)
    if ii > n_item:
        output('corrupted row?', line, ii, n_item)
        raise ValueError

    return vals

def write_dict_hdf5(filename, adict, level=0, group=None, fd=None):

    if level == 0:
        fd = pt.openFile(filename, mode='w', title='Recursive dict dump')
        group = '/'

    for key, val in adict.iteritems():
        if isinstance(val, dict):
            group2 = fd.createGroup(group, '_' + str(key), '%s group' % key)
            write_dict_hdf5(filename, val, level + 1, group2, fd)
        else:
            fd.createArray(group, '_' + str(key), val, '%s data' % key)

    if level == 0:
        fd.close()

def read_dict_hdf5(filename, level=0, group=None, fd=None):
    out = {}

    if level == 0:
        fd = pt.openFile(filename, mode='r')
        group = fd.root

    for name, gr in group._v_groups.iteritems():
        name = name.replace('_', '', 1)
        out[name] = read_dict_hdf5(filename, level + 1, gr, fd)

    for name, data in group._v_leaves.iteritems():
        name = name.replace('_', '', 1)
        out[name] = data.read()

    if level == 0:
        fd.close()

    return out

##
# 02.07.2007, c
def write_sparse_matrix_hdf5(filename, mtx, name='a sparse matrix'):
    """Assume CSR/CSC."""
    fd = pt.openFile(filename, mode='w', title=name)
    try:
        info = fd.createGroup('/', 'info')
        fd.createArray(info, 'dtype', mtx.dtype.str)
        fd.createArray(info, 'shape', mtx.shape)
        fd.createArray(info, 'format', mtx.format)

        data = fd.createGroup('/', 'data')
        fd.createArray(data, 'data', mtx.data)
        fd.createArray(data, 'indptr', mtx.indptr)
        fd.createArray(data, 'indices', mtx.indices)

    except:
        print 'matrix must be in SciPy sparse CSR/CSC format!'
        print mtx.__repr__()
        raise

    fd.close()

##
# 02.07.2007, c
# 08.10.2007
def read_sparse_matrix_hdf5(filename, output_format=None):
    import scipy.sparse as sp
    constructors = {'csr' : sp.csr_matrix, 'csc' : sp.csc_matrix}

    fd = pt.openFile(filename, mode='r')
    info = fd.root.info
    data = fd.root.data

    format = info.format.read()
    if not isinstance(format, str):
        format = format[0]

    dtype = info.dtype.read()
    if not isinstance(dtype, str):
        dtype = dtype[0]

    if output_format is None:
        constructor = constructors[format]
    else:
        constructor = constructors[output_format]

    if format in ['csc', 'csr']:
        mtx = constructor((data.data.read(),
                           data.indices.read(), data.indptr.read()),
                          shape=info.shape.read(), dtype=dtype)
    elif format == 'coo':
        mtx = constructor((data.data.read(),
                           nm.c_[data.rows.read(), data.cols.read()].T),
                          shape=info.shape.read(), dtype=dtype)
    else:
        print format
        raise ValueError
    fd.close()

    if output_format in ['csc', 'csr']:
        mtx.sort_indices()

    return mtx
<commit_msg>fix ensure_path() for zero-length path<commit_after>import numpy as nm
import os
import os.path as op
import fnmatch
import shutil
from base import output, Struct
try:
    import tables as pt
except:
    pt = None

class InDir(Struct):
    """
    Store the directory name a file is in, and prepend this name to other
    files.

    Examples
    --------

    >>> indir = InDir('output/file1')
    >>> print indir('file2')
    """
    def __init__(self, filename):
        self.dir = op.split(op.join(os.getcwd(), filename))[0]

    def __call__(self, filename):
        return op.join(self.dir, filename)

def ensure_path(filename):
    """
    Check if path to `filename` exists and if not, create the necessary
    intermediate directories.
    """
    dirname = os.path.dirname(filename)
    if dirname and not os.path.exists(dirname):
        os.makedirs(dirname)

def locate_files(pattern, root_dir=os.curdir):
    """
    Locate all files matching fiven filename pattern in and below
    supplied root directory.
    """
    for dirpath, dirnames, filenames in os.walk(os.path.abspath(root_dir)):
        for filename in fnmatch.filter(filenames, pattern):
            yield os.path.join(dirpath, filename)

def remove_files(root_dir):
    """
    Remove all files and directories in supplied root directory.
    """
    for dirpath, dirnames, filenames in os.walk(os.path.abspath(root_dir)):
        for filename in filenames:
            os.remove(os.path.join(root_dir, filename))

        for dirname in dirnames:
            shutil.rmtree(os.path.join(root_dir, dirname))

##
# 27.04.2006, c
def get_trunk(filename):
    return op.splitext(op.basename(filename))[0]

def get_print_info(n_step, fill=None):
    """
    Returns the max. number of digits in range(n_step) and the corresponding
    format string.

    Examples:

    >>> get_print_info(11)
    (2, '%2d')
    >>> get_print_info(8)
    (1, '%1d')
    >>> get_print_info(100)
    (2, '%2d')
    >>> get_print_info(101)
    (3, '%3d')
    >>> get_print_info(101, fill='0')
    (3, '%03d')
    """
    if n_step > 1:
        n_digit = int(nm.log10(n_step - 1) + 1)
        if fill is None:
            format = '%%%dd' % n_digit
        else:
            format = '%%%s%dd' % (fill, n_digit)
    else:
        n_digit, format = 0, None
    return n_digit, format

def skip_read_line(fd, no_eof=False):
    """
    Read the first non-empty line (if any) from the given file
    object. Return an empty string at EOF, if `no_eof` is False. If it
    is True, raise the EOFError instead.
    """
    ls = ''
    while 1:
        try:
            line = fd.readline()

        except EOFError:
            break

        if not line:
            if no_eof:
                raise EOFError

            else:
                break

        ls = line.strip()
        if ls and (ls[0] != '#'):
            break

    return ls

def read_token(fd):
    """
    Read a single token (sequence of non-whitespace characters) from the
    given file object.

    Notes
    -----
    Consumes the first whitespace character after the token.
    """
    out = ''
    # Skip initial whitespace.

    while 1:
        ch = fd.read(1)
        if ch.isspace(): continue
        elif len(ch) == 0: return out
        else: break

    while not ch.isspace():
        out = out + ch
        ch = fd.read(1)
        if len(ch) == 0: break

    return out

def read_array(fd, n_row, n_col, dtype):
    """
    Read a NumPy array of shape `(n_row, n_col)` from the given file
    object and cast it to type `dtype`.
    """
    count = n_row * n_col
    val = nm.fromfile(fd, sep=' ', count=count)

    if val.shape[0] < count:
        raise ValueError('(%d, %d) array reading failed!' % (n_row, n_col))

    val = nm.asarray(val, dtype=dtype)
    val.shape = (n_row, n_col)

    return val

##
# c: 05.02.2008, r: 05.02.2008
def read_list(fd, n_item, dtype):
    vals = []
    ii = 0
    while ii < n_item:
        line = [dtype(ic) for ic in fd.readline().split()]
        vals.append(line)
        ii += len(line)
    if ii > n_item:
        output('corrupted row?', line, ii, n_item)
        raise ValueError

    return vals

def write_dict_hdf5(filename, adict, level=0, group=None, fd=None):

    if level == 0:
        fd = pt.openFile(filename, mode='w', title='Recursive dict dump')
        group = '/'

    for key, val in adict.iteritems():
        if isinstance(val, dict):
            group2 = fd.createGroup(group, '_' + str(key), '%s group' % key)
            write_dict_hdf5(filename, val, level + 1, group2, fd)
        else:
            fd.createArray(group, '_' + str(key), val, '%s data' % key)

    if level == 0:
        fd.close()

def read_dict_hdf5(filename, level=0, group=None, fd=None):
    out = {}

    if level == 0:
        fd = pt.openFile(filename, mode='r')
        group = fd.root

    for name, gr in group._v_groups.iteritems():
        name = name.replace('_', '', 1)
        out[name] = read_dict_hdf5(filename, level + 1, gr, fd)

    for name, data in group._v_leaves.iteritems():
        name = name.replace('_', '', 1)
        out[name] = data.read()

    if level == 0:
        fd.close()

    return out

##
# 02.07.2007, c
def write_sparse_matrix_hdf5(filename, mtx, name='a sparse matrix'):
    """Assume CSR/CSC."""
    fd = pt.openFile(filename, mode='w', title=name)
    try:
        info = fd.createGroup('/', 'info')
        fd.createArray(info, 'dtype', mtx.dtype.str)
        fd.createArray(info, 'shape', mtx.shape)
        fd.createArray(info, 'format', mtx.format)

        data = fd.createGroup('/', 'data')
        fd.createArray(data, 'data', mtx.data)
        fd.createArray(data, 'indptr', mtx.indptr)
        fd.createArray(data, 'indices', mtx.indices)

    except:
        print 'matrix must be in SciPy sparse CSR/CSC format!'
        print mtx.__repr__()
        raise

    fd.close()

##
# 02.07.2007, c
# 08.10.2007
def read_sparse_matrix_hdf5(filename, output_format=None):
    import scipy.sparse as sp
    constructors = {'csr' : sp.csr_matrix, 'csc' : sp.csc_matrix}

    fd = pt.openFile(filename, mode='r')
    info = fd.root.info
    data = fd.root.data

    format = info.format.read()
    if not isinstance(format, str):
        format = format[0]

    dtype = info.dtype.read()
    if not isinstance(dtype, str):
        dtype = dtype[0]

    if output_format is None:
        constructor = constructors[format]
    else:
        constructor = constructors[output_format]

    if format in ['csc', 'csr']:
        mtx = constructor((data.data.read(),
                           data.indices.read(), data.indptr.read()),
                          shape=info.shape.read(), dtype=dtype)
    elif format == 'coo':
        mtx = constructor((data.data.read(),
                           nm.c_[data.rows.read(), data.cols.read()].T),
                          shape=info.shape.read(), dtype=dtype)
    else:
        print format
        raise ValueError
    fd.close()

    if output_format in ['csc', 'csr']:
        mtx.sort_indices()

    return mtx
<|endoftext|>


<commit_before>import filecmp
import os
import tempfile
import shutil
import time

import pytest

from synapseclient import File
from synapseclient.core.exceptions import SynapseMd5MismatchError
import synapseclient.core.utils as utils

def test_download_check_md5(syn, project, schedule_for_cleanup):
    tempfile_path = utils.make_bogus_data_file()
    schedule_for_cleanup(tempfile_path)
    entity_bad_md5 = syn.store(File(path=tempfile_path, parent=project['id'], synapseStore=False))

    pytest.raises(SynapseMd5MismatchError, syn._download_from_URL, entity_bad_md5['externalURL'], tempfile.gettempdir(),
                  entity_bad_md5['dataFileHandleId'], expected_md5="2345a")

def test_resume_partial_download(syn, project, schedule_for_cleanup):
    original_file = utils.make_bogus_data_file(40000)

    entity = File(original_file, parent=project['id'])
    entity = syn.store(entity)

    # stash the original file for comparison later
    shutil.move(original_file, original_file+'.original')
    original_file += '.original'
    schedule_for_cleanup(original_file)

    temp_dir = tempfile.gettempdir()

    url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id)
    path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
                                  expected_md5=entity.md5)

    # simulate an imcomplete download by putting the
    # complete file back into its temporary location
    tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId)
    shutil.move(path, tmp_path)

    # ...and truncating it to some fraction of its original size
    with open(tmp_path, 'r+') as f:
        f.truncate(3*os.path.getsize(original_file)//7)

    # this should complete the partial download
    path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
                                  expected_md5=entity.md5)

    assert filecmp.cmp(original_file, path), "File comparison failed"

def test_http_download__range_request_error(syn, project):
    # SYNPY-525

    file_path = utils.make_bogus_data_file()
    file_entity = syn.store(File(file_path, parent=project))

    syn.cache.purge(time.time())
    # download once and rename to temp file to simulate range exceed
    file_entity = syn.get(file_entity)
    shutil.move(file_entity.path, utils.temp_download_filename(file_entity.path, file_entity.dataFileHandleId))
    file_entity = syn.get(file_entity)

    assert file_path != file_entity.path
    assert filecmp.cmp(file_path, file_entity.path)
<commit_msg>fix cache.purge method for integration teets under multi CPU<commit_after>import filecmp
import os
import tempfile
import shutil
import time

import pytest

from synapseclient import File
from synapseclient.core.exceptions import SynapseMd5MismatchError
import synapseclient.core.utils as utils

def test_download_check_md5(syn, project, schedule_for_cleanup):
    tempfile_path = utils.make_bogus_data_file()
    schedule_for_cleanup(tempfile_path)
    entity_bad_md5 = syn.store(File(path=tempfile_path, parent=project['id'], synapseStore=False))

    pytest.raises(SynapseMd5MismatchError, syn._download_from_URL, entity_bad_md5['externalURL'], tempfile.gettempdir(),
                  entity_bad_md5['dataFileHandleId'], expected_md5="2345a")

def test_resume_partial_download(syn, project, schedule_for_cleanup):
    original_file = utils.make_bogus_data_file(40000)

    entity = File(original_file, parent=project['id'])
    entity = syn.store(entity)

    # stash the original file for comparison later
    shutil.move(original_file, original_file+'.original')
    original_file += '.original'
    schedule_for_cleanup(original_file)

    temp_dir = tempfile.gettempdir()

    url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id)
    path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
                                  expected_md5=entity.md5)

    # simulate an imcomplete download by putting the
    # complete file back into its temporary location
    tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId)
    shutil.move(path, tmp_path)

    # ...and truncating it to some fraction of its original size
    with open(tmp_path, 'r+') as f:
        f.truncate(3*os.path.getsize(original_file)//7)

    # this should complete the partial download
    path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
                                  expected_md5=entity.md5)

    assert filecmp.cmp(original_file, path), "File comparison failed"

def test_http_download__range_request_error(syn, project):
    # SYNPY-525

    file_path = utils.make_bogus_data_file()
    file_entity = syn.store(File(file_path, parent=project))

    syn.cache.remove(file_entity['_file_handle']['id'])
    # download once and rename to temp file to simulate range exceed
    file_entity = syn.get(file_entity)
    shutil.move(file_entity.path, utils.temp_download_filename(file_entity.path, file_entity.dataFileHandleId))
    file_entity = syn.get(file_entity)

    assert file_path != file_entity.path
    assert filecmp.cmp(file_path, file_entity.path)
<|endoftext|>


<commit_before>import numpy as nm
import scipy.sparse as sps

from sfepy.base.base import get_default, output

def sym_tri_eigen(diags):
    """
    Compute eigenvalues of a symmetric tridiagonal matrix. Naive
    implementation!
    """
    n_row = diags.shape[1]
    shape = (n_row, n_row)

    aux = nm.empty((3, n_row + 1), dtype=diags.dtype)
    aux[:2, :-1] = diags
    aux[2, 1:] = diags[1]
    mtx = sps.dia_matrix((aux, (0, -1, 1)), shape=shape)

    eigs, _ = nm.linalg.eig(mtx.toarray())

    return eigs

def cg_eigen(mtx, rhs=None, precond=None, i_max=None, eps_a=1e-10,
             verbose=False):
    """
    Make several iterations of the conjugate gradients and estimate so
    the eigenvalues of a (sparse) SPD matrix (Lanczos algorithm).
    """
    n_row = mtx.shape[0]
    norm = nm.linalg.norm

    rhs = get_default(rhs, nm.random.rand(n_row))
    i_max = get_default(i_max, max(n_row, 1000))

    lambda_max = 0
    lambda_min = 0
    econd = 1.0

    x0 = nm.zeros_like(rhs)
    r = r0 = rhs

    # The diagonals (0, 1) in two rows.
    diags = nm.empty((2, i_max + 1), dtype=mtx.dtype)
    diags[0, 0] = 0

    if precond is None:
        z0 = r0

    else:
        z0 = precond * r0

    x = x0
    z = z0

    p = nm.zeros_like(rhs)
    beta = 0.0

    rho0 = nm.dot(z0, r0)
    norm_r0 = norm(r0);

    if verbose:
        output('%5d lambda: %e %e cond: %e |R|: %e\n' % (0, 0, 0, 0, norm_r0))

    norm_rs = [norm_r0]

    for ii in xrange(i_max):
        p = z + beta * p
        q = mtx * p

        alpha = rho0 / nm.dot(p, q)

        x = x + alpha * p
        r = r - alpha * q

        if precond is None:
            z = r

        else:
            z = precond * r

        norm_r = norm(r)
        norm_rs.append(norm_r)

        rho1 = nm.dot(z, r)
        beta = rho1 / rho0

        # Lanczos
        diags[0, ii] += 1.0 / alpha
        diags[1, ii] = - nm.sqrt(beta) / alpha
        diags[0, ii+1] = beta / alpha

        if verbose:
            if ii % 100:
                eigs = sym_tri_eigen(diags[:, :ii+1])
                lambda_max = eigs.max()
                lambda_min = eigs.min()
                econd = lambda_max / lambda_min
                output('%5d lambda: %e %e cond: %e |R|: %e\n'
                       % (ii, lambda_min, lambda_max, econd, norm_r))

        if (norm_r / norm_r0) < eps_a:
            if verbose:
                output('converged on %d iters, |Ri|/|R0|: %e, econd: %e\n'
                       % (ii, norm_r / norm_r0, econd))
            break

        rho0 = rho1

    eigs = sym_tri_eigen(diags[:, :ii+1])
    lambda_max = eigs.max()
    lambda_min = eigs.min()
    econd = lambda_max / lambda_min
    if verbose:
        output('min: %e  max: %e  cond: %e\n'
               % (lambda_min, lambda_max, econd))

    return x, ii, norm_rs, eigs
<commit_msg>new arpack_eigs(), update and rename cg_eigen() -> cg_eigs()<commit_after>import numpy as nm
import scipy.sparse as sps

from sfepy.base.base import get_default, output

def sym_tri_eigen(diags):
    """
    Compute eigenvalues of a symmetric tridiagonal matrix. Naive
    implementation!
    """
    n_row = diags.shape[1]
    shape = (n_row, n_row)

    aux = nm.empty((3, n_row + 1), dtype=diags.dtype)
    aux[:2, :-1] = diags
    aux[2, 1:] = diags[1]
    mtx = sps.dia_matrix((aux, (0, -1, 1)), shape=shape)

    eigs, _ = nm.linalg.eig(mtx.toarray())
    eigs.sort()

    return eigs

def cg_eigs(mtx, rhs=None, precond=None, i_max=None, eps_a=1e-10,
            verbose=False, report_step=10):
    """
    Make several iterations of the conjugate gradients and estimate so
    the eigenvalues of a (sparse) SPD matrix (Lanczos algorithm).

    Returns
    -------
    vec : array
        The approximate solution to the linear system.
    n_it : int
        The number of CG iterations used.
    norm_rs : array
        Convergence history of residual norms
    eigs : array
        The approximate eigenvalues sorted in ascending order.
    """
    n_row = mtx.shape[0]
    norm = nm.linalg.norm

    rhs = get_default(rhs, nm.random.rand(n_row))
    i_max = get_default(i_max, min(n_row, 100))

    lambda_max = 0
    lambda_min = 0
    econd = 1.0

    x0 = nm.zeros_like(rhs)
    r = r0 = rhs

    # The diagonals (0, 1) in two rows.
    diags = nm.empty((2, i_max + 1), dtype=mtx.dtype)
    diags[0, 0] = 0

    if precond is None:
        z0 = r0

    else:
        z0 = precond * r0

    x = x0
    z = z0

    p = nm.zeros_like(rhs)
    beta = 0.0

    rho0 = nm.dot(z0, r0)
    norm_r0 = norm(r0);

    if verbose:
        output('%5d lambda: %e %e cond: %e |R|: %e\n' % (0, 0, 0, 0, norm_r0))

    norm_rs = [norm_r0]

    for ii in xrange(i_max):
        p = z + beta * p
        q = mtx * p

        alpha = rho0 / nm.dot(p, q)

        x = x + alpha * p
        r = r - alpha * q

        if precond is None:
            z = r

        else:
            z = precond * r

        norm_r = norm(r)
        norm_rs.append(norm_r)

        rho1 = nm.dot(z, r)
        beta = rho1 / rho0

        # Lanczos
        diags[0, ii] += 1.0 / alpha
        diags[1, ii] = - nm.sqrt(beta) / alpha
        diags[0, ii+1] = beta / alpha

        if verbose:
            if (ii % report_step) == 1:
                eigs = sym_tri_eigen(diags[:, :ii+1])
                lambda_min, lambda_max = eigs[0], eigs[-1]
                econd = lambda_max / lambda_min
                output('%5d lambda: %e %e cond: %e |R|: %e\n'
                       % (ii, lambda_min, lambda_max, econd, norm_r))

        if (norm_r / norm_r0) < eps_a:
            if verbose:
                output('converged on %d iters, |Ri|/|R0|: %e, econd: %e\n'
                       % (ii, norm_r / norm_r0, econd))
            break

        rho0 = rho1

    eigs = sym_tri_eigen(diags[:, :ii+1])
    lambda_min, lambda_max = eigs[0], eigs[-1]
    econd = lambda_max / lambda_min
    if verbose:
        output('min: %e  max: %e  cond: %e\n'
               % (lambda_min, lambda_max, econd))

    return x, ii, nm.array(norm_rs), eigs

def arpack_eigs(mtx, nev=1, which='SM'):
    """
    Calculate several eigenvalues and corresponding eigenvectors of a
    matrix using ARPACK from SciPy. The eigenvalues are sorted in
    ascending order.
    """
    from scipy.sparse.linalg.interface import aslinearoperator
    from scipy.sparse.linalg.eigen.arpack import speigs

    matvec = aslinearoperator(mtx).matvec
    eigs, vecs = speigs.ARPACK_eigs(matvec, mtx.shape[0], nev=nev, which=which)

    ii = nm.argsort(eigs)
    eigs = eigs[ii]
    vecs = vecs[:,ii]

    return eigs, vecs
<|endoftext|>


<commit_before>from django.db.models import Prefetch
from django.views.generic import (TemplateView, DetailView, FormView,
                                  ListView, UpdateView)

from candidates.views.version_data import get_client_ip
from candidates.models import LoggedAction
from elections.models import Election

from ..constants import CONFIRMED_STATUS
from ..models import CouncilElection, CouncilElectionResultSet
from ..forms import (ReportCouncilElectionControlForm,
                     ReviewControlForm)
from .base import BaseResultsViewMixin

class CouncilsWithElections(BaseResultsViewMixin, TemplateView):
    template_name = "uk_results/councils_with_elections.html"

    def get_context_data(self, **kwargs):
        context = super(CouncilsWithElections, self).get_context_data(**kwargs)
        councils = CouncilElection.objects.all().order_by('council__name')
        councils = councils.select_related(
            'council',
            'election',
            # 'reported_results',
        )
        councils = councils.prefetch_related(
            Prefetch(
                'reported_results',
                CouncilElectionResultSet.objects.select_related(
                    'council_election',
                    'council_election__election',
                    'council_election__council',
                )
            )
        )
        context['council_elections'] = councils

        return context

class CouncilElectionView(BaseResultsViewMixin, DetailView):
    model = CouncilElection

    def get_object(self):
        election_id = self.kwargs.get('election_id')
        council_election = CouncilElection.objects.select_related(
            'election',
            'council',
        )
        council_election = council_election.get(election__slug=election_id)
        return council_election

    def get_context_data(self, **kwargs):
        context = super(CouncilElectionView, self).get_context_data(**kwargs)
        context['posts'] = self.object.election.posts.select_related(
            'base',
            'base__area',
        ).order_by('base__label')
        return context

class ReportCouncilElectionView(BaseResultsViewMixin, FormView):
    model = CouncilElection
    pk_url_kwarg = 'council_election'
    template_name = "uk_results/report_council_election_control.html"

    def get_context_data(self, **kwargs):
        context = super(ReportCouncilElectionView, self).get_context_data(**kwargs)
        context['object'] = self.object
        return context

    def get_form(self, form_class=None):
        """
        Returns an instance of the form to be used in this view.
        """
        self.object = self.model.objects.get(
            election__slug=self.kwargs['election_id']
        )

        return ReportCouncilElectionControlForm(
            council_election=self.object,
            **self.get_form_kwargs()
        )

    def get_success_url(self):
        return self.object.get_absolute_url()

    def form_valid(self, form):
        instance = form.save()
        LoggedAction.objects.create(
            user=self.request.user,
            action_type='record-council-control',
            ip_address=get_client_ip(self.request),
            source=form['source'].value()
        )
        if 'report_and_confirm' in self.request.POST:
            instance.review_status = CONFIRMED_STATUS
            instance.reviewed_by = self.request.user
            instance.save()

            LoggedAction.objects.create(
                user=self.request.user,
                action_type='confirm-council-control',
                ip_address=get_client_ip(self.request),
                source="Confirmed when reporting",
            )

        return super(ReportCouncilElectionView, self).form_valid(form)

class LatestControlResults(BaseResultsViewMixin, ListView):
    template_name = "uk_results/latest_control_results.html"
    queryset = CouncilElectionResultSet.objects.all()

    def get_queryset(self):
        queryset = self.queryset
        status = self.request.GET.get('status')
        if status:
            if status == "confirmed":
                queryset = queryset.confirmed()
            if status == "unconfirmed":
                queryset = queryset.unconfirmed()
            if status == "rejected":
                queryset = queryset.rejected()
        return queryset

class ConfirmControl(BaseResultsViewMixin, UpdateView):
    template_name = "uk_results/review_reported_control.html"
    queryset = CouncilElectionResultSet.objects.all()

    def get_form(self, form_class=None):
        kwargs = self.get_form_kwargs()
        kwargs['initial'].update({'reviewed_by': self.request.user})
        return ReviewControlForm(
            **kwargs
        )

    def get_success_url(self):
        return self.object.council_election.get_absolute_url()

    def form_valid(self, form):
        form.save()
        LoggedAction.objects.create(
            user=self.request.user,
            action_type='confirm-council-control',
            ip_address=get_client_ip(self.request),
            source=form['review_source'].value()
        )
        return super(ConfirmControl, self).form_valid(form)
<commit_msg>Don't log actions for non-auth users<commit_after>from django.db.models import Prefetch
from django.views.generic import (TemplateView, DetailView, FormView,
                                  ListView, UpdateView)

from candidates.views.version_data import get_client_ip
from candidates.models import LoggedAction
from elections.models import Election

from ..constants import CONFIRMED_STATUS
from ..models import CouncilElection, CouncilElectionResultSet
from ..forms import (ReportCouncilElectionControlForm,
                     ReviewControlForm)
from .base import BaseResultsViewMixin

class CouncilsWithElections(BaseResultsViewMixin, TemplateView):
    template_name = "uk_results/councils_with_elections.html"

    def get_context_data(self, **kwargs):
        context = super(CouncilsWithElections, self).get_context_data(**kwargs)
        councils = CouncilElection.objects.all().order_by('council__name')
        councils = councils.select_related(
            'council',
            'election',
            # 'reported_results',
        )
        councils = councils.prefetch_related(
            Prefetch(
                'reported_results',
                CouncilElectionResultSet.objects.select_related(
                    'council_election',
                    'council_election__election',
                    'council_election__council',
                )
            )
        )
        context['council_elections'] = councils

        return context

class CouncilElectionView(BaseResultsViewMixin, DetailView):
    model = CouncilElection

    def get_object(self):
        election_id = self.kwargs.get('election_id')
        council_election = CouncilElection.objects.select_related(
            'election',
            'council',
        )
        council_election = council_election.get(election__slug=election_id)
        return council_election

    def get_context_data(self, **kwargs):
        context = super(CouncilElectionView, self).get_context_data(**kwargs)
        context['posts'] = self.object.election.posts.select_related(
            'base',
            'base__area',
        ).order_by('base__label')
        return context

class ReportCouncilElectionView(BaseResultsViewMixin, FormView):
    model = CouncilElection
    pk_url_kwarg = 'council_election'
    template_name = "uk_results/report_council_election_control.html"

    def get_context_data(self, **kwargs):
        context = super(ReportCouncilElectionView, self).get_context_data(**kwargs)
        context['object'] = self.object
        return context

    def get_form(self, form_class=None):
        """
        Returns an instance of the form to be used in this view.
        """
        self.object = self.model.objects.get(
            election__slug=self.kwargs['election_id']
        )

        return ReportCouncilElectionControlForm(
            council_election=self.object,
            **self.get_form_kwargs()
        )

    def get_success_url(self):
        return self.object.get_absolute_url()

    def form_valid(self, form):
        instance = form.save()
        user = None
        if self.request.user.is_authenticated():
            user = self.request.user
            LoggedAction.objects.create(
                user=user,
                action_type='record-council-control',
                ip_address=get_client_ip(self.request),
                source=form['source'].value()
            )
        if 'report_and_confirm' in self.request.POST:
            instance.review_status = CONFIRMED_STATUS
            instance.reviewed_by = self.request.user
            instance.save()

            LoggedAction.objects.create(
                user=self.request.user,
                action_type='confirm-council-control',
                ip_address=get_client_ip(self.request),
                source="Confirmed when reporting",
            )

        return super(ReportCouncilElectionView, self).form_valid(form)

class LatestControlResults(BaseResultsViewMixin, ListView):
    template_name = "uk_results/latest_control_results.html"
    queryset = CouncilElectionResultSet.objects.all()

    def get_queryset(self):
        queryset = self.queryset
        status = self.request.GET.get('status')
        if status:
            if status == "confirmed":
                queryset = queryset.confirmed()
            if status == "unconfirmed":
                queryset = queryset.unconfirmed()
            if status == "rejected":
                queryset = queryset.rejected()
        return queryset

class ConfirmControl(BaseResultsViewMixin, UpdateView):
    template_name = "uk_results/review_reported_control.html"
    queryset = CouncilElectionResultSet.objects.all()

    def get_form(self, form_class=None):
        kwargs = self.get_form_kwargs()
        kwargs['initial'].update({'reviewed_by': self.request.user})
        return ReviewControlForm(
            **kwargs
        )

    def get_success_url(self):
        return self.object.council_election.get_absolute_url()

    def form_valid(self, form):
        form.save()
        LoggedAction.objects.create(
            user=self.request.user,
            action_type='confirm-council-control',
            ip_address=get_client_ip(self.request),
            source=form['review_source'].value()
        )
        return super(ConfirmControl, self).form_valid(form)
<|endoftext|>