<commit_before># -*- coding: utf-8 -*-
"""
This module implements LaTeX base classes that can be subclassed.

..  :copyright: (c) 2014 by Jelte Fennema.
    :license: MIT, see License for more details.
"""

from collections import UserList
from pylatex.utils import dumps_list
from contextlib import contextmanager
from .latex_object import LatexObject
from .command import Command, Arguments

class Container(LatexObject, UserList):
    """A base class that groups multiple LaTeX classes.

    This class should be subclassed when a LaTeX class has content that is of
    variable length. It subclasses UserList, so it holds a list of elements
    that can simply be accessed by using normal list functionality, like
    indexing or appending.

    """

    content_separator = '%\n'

    def __init__(self, *, data=None):
        r"""
        Args
        ----
        data: list, `~.LatexObject` or something that can be converted to a \
                string
            The content with which the container is initialized
        """

        if data is None:
            data = []
        elif not isinstance(data, list):
            # If the data is not already a list make it a list, otherwise list
            # operations will not work
            data = [data]

        self.data = data
        self.real_data = data  # Always the data of this instance

        super().__init__()

    @property
    def _repr_attributes(self):
        return super()._repr_attributes + ['real_data']

    def dumps_content(self, **kwargs):
        r"""Represent the container as a string in LaTeX syntax.

        Args
        ----
        \*\*kwargs:
            Arguments that can be passed to `~.dumps_list`

        Returns
        -------
        string:
            A LaTeX string representing the container
        """

        return dumps_list(self, escape=self.escape,
                          token=self.content_separator, **kwargs)

    def _propagate_packages(self):
        """Make sure packages get propagated."""

        for item in self.data:
            if isinstance(item, LatexObject):
                if isinstance(item, Container):
                    item._propagate_packages()
                for p in item.packages:
                    self.packages.add(p)

    def dumps_packages(self):
        r"""Represent the packages needed as a string in LaTeX syntax.

        Returns
        -------
        string:
            A LaTeX string representing the packages of the container
        """

        self._propagate_packages()

        return super().dumps_packages()

    @contextmanager
    def create(self, child):
        """Add a LaTeX object to current container, context-manager style.

        Args
        ----
        child: `~.Container`
            An object to be added to the current container
        """

        prev_data = self.data
        self.data = child.data  # This way append works appends to the child

        yield child  # allows with ... as to be used as well

        self.data = prev_data
        self.append(child)

class Environment(Container):
    r"""A base class for LaTeX environments.

    This class implements the basics of a LaTeX environment. A LaTeX
    environment looks like this:

    .. code-block:: latex

        \begin{environment_name}
            Some content that is in the environment
        \end{environment_name}

    The text that is used in the place of environment_name is by default the
    name of the class in lowercase.
    However, this default can be overridden in 2 ways:
        1. setting the _latex_name class variable when declaring the class
        2. setting the _latex_name attribute when initialising object
    """

    #: Set to true if this full container should be equivalent to an empty
    #: string if it has no content.
    omit_if_empty = False

    def __init__(self, *, options=None, arguments=None, start_arguments=None,
                 **kwargs):
        r"""
        Args
        ----
        options: str or list or  `~.Options`
            Options to be added to the ``\begin`` command

        arguments: str or list or `~.Arguments`
            Arguments to be added to the ``\begin`` command

        start_arguments: str or list or `~.Arguments`
            Arguments to be added before the options
        """

        self.options = options
        self.arguments = arguments
        self.start_arguments = start_arguments

        super().__init__(**kwargs)

    def dumps(self):
        """Represent the environment as a string in LaTeX syntax.

        Returns
        -------
        str
            A LaTeX string representing the environment.
        """

        content = self.dumps_content()
        if not content.strip() and self.omit_if_empty:
            return ''

        string = ''

        # Something other than None needs to be used as extra arguments, that
        # way the options end up behind the latex_name argument.
        if self.arguments is None:
            extra_arguments = Arguments()
        else:
            extra_arguments = self.arguments

        begin = Command('begin', self.start_arguments, self.options,
                        extra_arguments=extra_arguments)
        begin.arguments._positional_args.insert(0, self.latex_name)
        string += begin.dumps() + '%\n'

        string += content + '%\n'

        string += Command('end', self.latex_name).dumps()

        return string

class ContainerCommand(Container):
    r"""A base class for a container command (A command which contains data).

    Container command example:

    .. code-block:: latex

        \CommandName[options]{arguments}{
            data
        }

    """

    omit_if_empty = False

    def __init__(self, arguments=None, options=None, *, data=None, **kwargs):
        r"""
        Args
        ----
        arguments: str or `list`
            The arguments for the container command
        options: str, list or `~.Options`
            The options for the preamble command
        data: str or `~.LatexObject`
            The data to place inside the preamble command
        """

        self.arguments = arguments
        self.options = options

        super().__init__(data=data, **kwargs)

    def dumps(self):
        r"""Convert the container to a string in latex syntax."""

        content = self.dumps_content()

        if not content.strip() and self.omit_if_empty:
            return ''

        string = ''

        start = Command(self.latex_name, arguments=self.arguments,
                        options=self.options)

        string += start.dumps() + '{ \n'

        if content != '':
            string += content + '\n}'
        else:
            string += '}'

        return string
<commit_msg>indenting docstring fix<commit_after># -*- coding: utf-8 -*-
"""
This module implements LaTeX base classes that can be subclassed.

..  :copyright: (c) 2014 by Jelte Fennema.
    :license: MIT, see License for more details.
"""

from collections import UserList
from pylatex.utils import dumps_list
from contextlib import contextmanager
from .latex_object import LatexObject
from .command import Command, Arguments

class Container(LatexObject, UserList):
    """A base class that groups multiple LaTeX classes.

    This class should be subclassed when a LaTeX class has content that is of
    variable length. It subclasses UserList, so it holds a list of elements
    that can simply be accessed by using normal list functionality, like
    indexing or appending.

    """

    content_separator = '%\n'

    def __init__(self, *, data=None):
        r"""
        Args
        ----
        data: list, `~.LatexObject` or something that can be converted to a \
                string
            The content with which the container is initialized
        """

        if data is None:
            data = []
        elif not isinstance(data, list):
            # If the data is not already a list make it a list, otherwise list
            # operations will not work
            data = [data]

        self.data = data
        self.real_data = data  # Always the data of this instance

        super().__init__()

    @property
    def _repr_attributes(self):
        return super()._repr_attributes + ['real_data']

    def dumps_content(self, **kwargs):
        r"""Represent the container as a string in LaTeX syntax.

        Args
        ----
        \*\*kwargs:
            Arguments that can be passed to `~.dumps_list`

        Returns
        -------
        string:
            A LaTeX string representing the container
        """

        return dumps_list(self, escape=self.escape,
                          token=self.content_separator, **kwargs)

    def _propagate_packages(self):
        """Make sure packages get propagated."""

        for item in self.data:
            if isinstance(item, LatexObject):
                if isinstance(item, Container):
                    item._propagate_packages()
                for p in item.packages:
                    self.packages.add(p)

    def dumps_packages(self):
        r"""Represent the packages needed as a string in LaTeX syntax.

        Returns
        -------
        string:
            A LaTeX string representing the packages of the container
        """

        self._propagate_packages()

        return super().dumps_packages()

    @contextmanager
    def create(self, child):
        """Add a LaTeX object to current container, context-manager style.

        Args
        ----
        child: `~.Container`
            An object to be added to the current container
        """

        prev_data = self.data
        self.data = child.data  # This way append works appends to the child

        yield child  # allows with ... as to be used as well

        self.data = prev_data
        self.append(child)

class Environment(Container):
    r"""A base class for LaTeX environments.

    This class implements the basics of a LaTeX environment. A LaTeX
    environment looks like this:

    .. code-block:: latex

        \begin{environment_name}
            Some content that is in the environment
        \end{environment_name}

    The text that is used in the place of environment_name is by default the
    name of the class in lowercase.
    However, this default can be overridden in 2 ways:
    1. setting the _latex_name class variable when declaring the class
    2. setting the _latex_name attribute when initialising object
    """

    #: Set to true if this full container should be equivalent to an empty
    #: string if it has no content.
    omit_if_empty = False

    def __init__(self, *, options=None, arguments=None, start_arguments=None,
                 **kwargs):
        r"""
        Args
        ----
        options: str or list or  `~.Options`
            Options to be added to the ``\begin`` command

        arguments: str or list or `~.Arguments`
            Arguments to be added to the ``\begin`` command

        start_arguments: str or list or `~.Arguments`
            Arguments to be added before the options
        """

        self.options = options
        self.arguments = arguments
        self.start_arguments = start_arguments

        super().__init__(**kwargs)

    def dumps(self):
        """Represent the environment as a string in LaTeX syntax.

        Returns
        -------
        str
            A LaTeX string representing the environment.
        """

        content = self.dumps_content()
        if not content.strip() and self.omit_if_empty:
            return ''

        string = ''

        # Something other than None needs to be used as extra arguments, that
        # way the options end up behind the latex_name argument.
        if self.arguments is None:
            extra_arguments = Arguments()
        else:
            extra_arguments = self.arguments

        begin = Command('begin', self.start_arguments, self.options,
                        extra_arguments=extra_arguments)
        begin.arguments._positional_args.insert(0, self.latex_name)
        string += begin.dumps() + '%\n'

        string += content + '%\n'

        string += Command('end', self.latex_name).dumps()

        return string

class ContainerCommand(Container):
    r"""A base class for a container command (A command which contains data).

    Container command example:

    .. code-block:: latex

        \CommandName[options]{arguments}{
            data
        }

    """

    omit_if_empty = False

    def __init__(self, arguments=None, options=None, *, data=None, **kwargs):
        r"""
        Args
        ----
        arguments: str or `list`
            The arguments for the container command
        options: str, list or `~.Options`
            The options for the preamble command
        data: str or `~.LatexObject`
            The data to place inside the preamble command
        """

        self.arguments = arguments
        self.options = options

        super().__init__(data=data, **kwargs)

    def dumps(self):
        r"""Convert the container to a string in latex syntax."""

        content = self.dumps_content()

        if not content.strip() and self.omit_if_empty:
            return ''

        string = ''

        start = Command(self.latex_name, arguments=self.arguments,
                        options=self.options)

        string += start.dumps() + '{ \n'

        if content != '':
            string += content + '\n}'
        else:
            string += '}'

        return string
<|endoftext|>


<commit_before># pyOCD debugger
# Copyright (c) 2018 Arm Limited
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ..flash.flash import Flash
from ..core.coresight_target import CoreSightTarget
from ..core.memory_map import (FlashRegion, RamRegion, MemoryMap)
from ..debug.svd import SVDFile

FLASH_ALGO = {
    'load_address' : 0x20000000,

    # Flash algorithm as a hex string
    'instructions': [
    0xE00ABE00, 0x062D780D, 0x24084068, 0xD3000040, 0x1E644058, 0x1C49D1FA, 0x2A001E52, 0x4770D1F2,
    0xd0012a01, 0xd1172a02, 0x6981483b, 0x0212220f, 0x61814311, 0x60c14939, 0x60c14939, 0x61014939,
    0x61014939, 0x02c069c0, 0x4839d406, 0x60014937, 0x60412106, 0x60814937, 0x47702000, 0xd0012801,
    0xd1082802, 0x6841482c, 0x43112202, 0x68416041, 0x43112201, 0x20006041, 0xb5304770, 0x684a4926,
    0x4322154c, 0x684a604a, 0x432a2508, 0x2200604a, 0x48296002, 0xe0004a26, 0x698b6010, 0xd1fb07db,
    0x43a06848, 0x68486048, 0x604843a8, 0xbd302000, 0x47702001, 0x4c18b5f0, 0x15252300, 0x313f2608,
    0x468c0989, 0x6861e024, 0x60614329, 0x43316861, 0x21406061, 0xc080ca80, 0x29001f09, 0x4916d1fa,
    0x07ff69a7, 0x4f12d002, 0xe7f96039, 0x050969a1, 0xd0060f09, 0x210f69a0, 0x43080209, 0x200161a0,
    0x6861bdf0, 0x606143a9, 0x43b16861, 0x1c5b6061, 0xd8d8459c, 0xbdf02000, 0x40022000, 0x89abcdef,
    0x02030405, 0x8c9daebf, 0x13141516, 0x00005555, 0x40003000, 0x00000fff, 0x0000aaaa, 0x00000000
    ],

    # Relative function addresses
    'pc_init': 0x20000021,
    'pc_unInit': 0x2000005d,
    'pc_program_page': 0x200000b5,
    'pc_erase_sector': 0x2000007b,
    'pc_eraseAll': 0x12000001,

    'static_base' : 0x20000000 + 0x00000020 + 0x0000011c,
    'begin_stack' : 0x20000400,
    'begin_data' : 0x20000000 + 0x1000,
    'page_size' : 0x400,
    'analyzer_supported' : False,
    'analyzer_address' : 0x00000000,
    'page_buffers' : [0x20001000, 0x20001400],   # Enable double buffering
    'min_program_length' : 0x400,

    # Flash information
    'flash_start': 0x8000000,
    'flash_size': 0x8000,
    'sector_sizes': (
        (0x0, 0x80),
    )
}

class STM32L031x6(CoreSightTarget):

    memoryMap = MemoryMap(
        FlashRegion(start=0x08000000, length=0x8000, blocksize=0x1000,  is_boot_memory=True, algo=FLASH_ALGO),
        RamRegion(start=0x20000000, length=0x2000),
        FlashRegion(start=0x08080000, length=0x400, blocksize=0x400, algo=FLASH_ALGO)
        )

    def __init__(self, link):
        super(STM32L031x6, self).__init__(link, self.memoryMap)

    def create_init_sequence(self):
        seq = super(STM32L031x6, self).create_init_sequence()
        return seq
<commit_msg>Added initialisation of DBG_CR register, added names to flash and ram regions<commit_after># pyOCD debugger
# Copyright (c) 2018 Arm Limited
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ..flash.flash import Flash
from ..core.coresight_target import CoreSightTarget
from ..core.memory_map import (FlashRegion, RamRegion, MemoryMap)
from ..debug.svd import SVDFile

class DBGMCU:
    # en.DM00108282.pdf (STM32L0x1 Reference manual)
    # 27.9.3 Debug MCU configuration register (DBG_CR)
    DBG_CR = 0x40015804
    DBG_STANDBY = (1 << 2)
    DBG_STOP = (1 << 1)
    DBG_SLEEP = 1
    DBG_CR_VALUE = DBG_STANDBY | DBG_STOP | DBG_SLEEP

FLASH_ALGO = {
    'load_address' : 0x20000000,

    # Flash algorithm as a hex string
    'instructions': [
    0xE00ABE00, 0x062D780D, 0x24084068, 0xD3000040, 0x1E644058, 0x1C49D1FA, 0x2A001E52, 0x4770D1F2,
    0xd0012a01, 0xd1172a02, 0x6981483b, 0x0212220f, 0x61814311, 0x60c14939, 0x60c14939, 0x61014939,
    0x61014939, 0x02c069c0, 0x4839d406, 0x60014937, 0x60412106, 0x60814937, 0x47702000, 0xd0012801,
    0xd1082802, 0x6841482c, 0x43112202, 0x68416041, 0x43112201, 0x20006041, 0xb5304770, 0x684a4926,
    0x4322154c, 0x684a604a, 0x432a2508, 0x2200604a, 0x48296002, 0xe0004a26, 0x698b6010, 0xd1fb07db,
    0x43a06848, 0x68486048, 0x604843a8, 0xbd302000, 0x47702001, 0x4c18b5f0, 0x15252300, 0x313f2608,
    0x468c0989, 0x6861e024, 0x60614329, 0x43316861, 0x21406061, 0xc080ca80, 0x29001f09, 0x4916d1fa,
    0x07ff69a7, 0x4f12d002, 0xe7f96039, 0x050969a1, 0xd0060f09, 0x210f69a0, 0x43080209, 0x200161a0,
    0x6861bdf0, 0x606143a9, 0x43b16861, 0x1c5b6061, 0xd8d8459c, 0xbdf02000, 0x40022000, 0x89abcdef,
    0x02030405, 0x8c9daebf, 0x13141516, 0x00005555, 0x40003000, 0x00000fff, 0x0000aaaa, 0x00000000
    ],

    # Relative function addresses
    'pc_init': 0x20000021,
    'pc_unInit': 0x2000005d,
    'pc_program_page': 0x200000b5,
    'pc_erase_sector': 0x2000007b,
    'pc_eraseAll': 0x12000001,

    'static_base' : 0x20000000 + 0x00000020 + 0x0000011c,
    'begin_stack' : 0x20000400,
    'begin_data' : 0x20000000 + 0x1000,
    'page_size' : 0x400,
    'analyzer_supported' : False,
    'analyzer_address' : 0x00000000,
    'page_buffers' : [0x20001000, 0x20001400],   # Enable double buffering
    'min_program_length' : 0x400,

    # Flash information
    'flash_start': 0x8000000,
    'flash_size': 0x8000,
    'sector_sizes': (
        (0x0, 0x80),
    )
}

class STM32L031x6(CoreSightTarget):

    memoryMap = MemoryMap(
        FlashRegion(name='Flash', start=0x08000000, length=0x8000, blocksize=0x1000,  is_boot_memory=True, algo=FLASH_ALGO),
        RamRegion(name='RAM', start=0x20000000, length=0x2000),
        FlashRegion(name='EEPROM', start=0x08080000, length=0x400, blocksize=0x400, algo=FLASH_ALGO)
        )

    def __init__(self, link):
        super(STM32L031x6, self).__init__(link, self.memoryMap)

    def create_init_sequence(self):
        seq = super(STM32L031x6, self).create_init_sequence()

        seq.insert_after('create_cores',
            ('setup_dbgmcu', self.setup_dbgmcu)
            )

        return seq

    def setup_dbgmcu(self):
        self.write32(DBGMCU.DBG_CR, DBGMCU.DBG_CR_VALUE)<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2011, Taher Haveliwala <oss@taherh.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#         * Redistributions of source code must retain the above copyright
#           notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above copyright
#           notice, this list of conditions and the following disclaimer in the
#           documentation and/or other materials provided with the distribution.
#         * The names of project contributors may not be used to endorse or
#           promote products derived from this software without specific
#           prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''
SimIndex

See :class:`pysimsearch.sim_index.MemorySimIndex` for sample usage

'''

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import abc
import io
import itertools

from .. import doc_reader
from .. import term_vec
from ..exceptions import *
from ..query_scorer import QueryScorer

class SimIndex(object):
    '''
    Base class for similarity indexes

    Defines interface as well as provides default implementation for
    several methods.

    Instance Attributes:
        config: dictionary of configuration variables

    '''

    __metaclass__ = abc.ABCMeta

    def __init__(self):
        self._config = {
            'lowercase': True,
            'stoplist': {}  # using dict instead of set, for rpc support
        }
        self.query_scorer = None
        self._N = 0
        self._global_N = None
        self._next_docid = 0

    def config(self, key):
        return self._config[key]

    def set_config(self, key, value):
        self._config[key] = value

    def update_config(self, **d):
        self._config.update(d)

    def load_stoplist(self, stopfile):
        stoplist = {}
        for line in stopfile:
            stoplist.update(zip(line.split(), itertools.repeat(1)))
        self.set_config('stoplist', stoplist)

    @abc.abstractmethod
    def set_global_df_map(self, df_map):
        '''Set global df stats'''
        return

    @abc.abstractmethod
    def get_local_df_map(self):
        '''Get local df stats'''
        return

    @abc.abstractmethod
    def get_name_to_docid_map(self):
        '''Return local mapping of name to docids'''
        return

    def set_global_N(self, N):
        '''Set global number of documents'''
        self._global_N = N

    def get_local_N(self):
        '''Return local number of documents'''
        return self._N

    def set_query_scorer(self, query_scorer):
        '''Set the query_scorer

        Params:
            query_scorer: if string type, we assume it is a scorer name,
                          else we assume it is itself a scoring object
                          of base type :class:`query_scorer.QueryScorer`.
        '''
        if isinstance(query_scorer, basestring):
            self.query_scorer = QueryScorer.make_scorer(query_scorer)
        else:
            self.query_scorer = query_scorer

    @abc.abstractmethod
    def index_files(self, named_files):
        '''Add ``named_files`` to the index

        Params:
            named_files: iterable of (filename, file) pairs.
                         Takes ownership of (and consumes) the files.
        '''
        return

    def index_filenames(self, *filenames):
        '''Add ``filenames`` to the index

        Convenience method that wraps :meth:`index_files()`

        Params:
            ``filenames``: list of filenames to add to the index.
        '''
        return self.index_files(doc_reader.get_text_files(filenames))

    def index_urls(self, *urls):
        '''Add ``urls`` to the index

        Convenience method that wraps :meth:`index_files()`

        Params:
            ``urls``: list of urls of web pages to add to the index.
        '''
        return self.index_files(doc_reader.get_urls(urls))

    def index_string_buffers(self, named_string_buffers):
        '''Add ``named_string_buffers`` to the index

        Params:
            named_string_buffers: iterable of (name, string) tuples, where
                                  the string contains the data to index.

        '''
        named_files = []
        for (name, string_buffer) in named_string_buffers:
            if isinstance(string_buffer, str):
                string_buffer = unicode(string_buffer)
            named_files.append((name, io.StringIO(string_buffer)))
        self.index_files(named_files)

    @abc.abstractmethod
    def del_docids(self, *docids):
        '''Deletes documents corresponding to docids from the index'''
        return

    @abc.abstractmethod
    def docid_to_name(self, docid):
        '''Returns document name for a given docid'''
        return

    @abc.abstractmethod
    def name_to_docid(self, name):
        '''Returns docid for a given document name'''
        return

    @abc.abstractmethod
    def postings_list(self, term):
        '''
        Return list of (docid, frequency) tuples for docs that contain term
        '''
        return

    def docids_with_terms(self, terms):
        '''Returns a list of docids of docs containing all terms'''
        docs = None  # will hold a set of matching docids
        for term in terms:
            if docs is None:
                docs = set((x[0] for x in self.postings_list(term)))
            else:
                docs.intersection_update(
                    (x[0] for x in self.postings_list(term)))

        # return sorted list
        if docs is None: docs = []
        return sorted(docs)

    def docnames_with_terms(self, *terms):
        '''Returns an iterable of docnames containing terms'''
        if self.config('lowercase'):
            terms = [term.lower() for term in terms]
        return (self.docid_to_name(docid) for docid in self.docids_with_terms(terms))

    def query(self, q):
        '''Finds documents similar to q.

        Params:
            query: the query given as either a string or query vector

        Returns:
            A iterable of (docname, score) tuples sorted by score
        '''
        if isinstance(q, basestring):
            if self.config('lowercase'):
                q = q.lower()
            if isinstance(q, str):
                q = unicode(q)
            return self._query(term_vec.term_vec(q))
        else:
            if self.config('lowercase'):
                q = {term.lower(): freq for (term, freq) in q.items()}
            return self._query(q)

    @abc.abstractmethod
    def _query(self, query_vec):
        '''Finds documents similar to query_vec

        Params:
            query_vec: term vector representing query document

        Returns:
            A iterable of (docname, score) tuples sorted by score
        '''
        return

<commit_msg>o lowercase-handling fixes for consistency<commit_after>#!/usr/bin/env python

# Copyright (c) 2011, Taher Haveliwala <oss@taherh.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#         * Redistributions of source code must retain the above copyright
#           notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above copyright
#           notice, this list of conditions and the following disclaimer in the
#           documentation and/or other materials provided with the distribution.
#         * The names of project contributors may not be used to endorse or
#           promote products derived from this software without specific
#           prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''
SimIndex

See :class:`pysimsearch.sim_index.MemorySimIndex` for sample usage

'''

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import abc
import io
import itertools

from .. import doc_reader
from .. import term_vec
from ..exceptions import *
from ..query_scorer import QueryScorer

class SimIndex(object):
    '''
    Base class for similarity indexes

    Defines interface as well as provides default implementation for
    several methods.

    Instance Attributes:
        config: dictionary of configuration variables

    '''

    __metaclass__ = abc.ABCMeta

    def __init__(self):
        self._config = {
            'lowercase': True,
            'stoplist': {}  # using dict instead of set, for rpc support
        }
        self.query_scorer = None
        self._N = 0
        self._global_N = None
        self._next_docid = 0

    def config(self, key):
        return self._config[key]

    def set_config(self, key, value):
        self._config[key] = value

    def update_config(self, **d):
        self._config.update(d)

    def load_stoplist(self, stopfile):
        stoplist = {}
        for line in stopfile:
            stoplist.update(zip(line.split(), itertools.repeat(1)))
        self.set_config('stoplist', stoplist)

    @abc.abstractmethod
    def set_global_df_map(self, df_map):
        '''Set global df stats'''
        return

    @abc.abstractmethod
    def get_local_df_map(self):
        '''Get local df stats'''
        return

    @abc.abstractmethod
    def get_name_to_docid_map(self):
        '''Return local mapping of name to docids'''
        return

    def set_global_N(self, N):
        '''Set global number of documents'''
        self._global_N = N

    def get_local_N(self):
        '''Return local number of documents'''
        return self._N

    def set_query_scorer(self, query_scorer):
        '''Set the query_scorer

        Params:
            query_scorer: if string type, we assume it is a scorer name,
                          else we assume it is itself a scoring object
                          of base type :class:`query_scorer.QueryScorer`.
        '''
        if isinstance(query_scorer, basestring):
            self.query_scorer = QueryScorer.make_scorer(query_scorer)
        else:
            self.query_scorer = query_scorer

    @abc.abstractmethod
    def index_files(self, named_files):
        '''Add ``named_files`` to the index

        Params:
            named_files: iterable of (filename, file) pairs.
                         Takes ownership of (and consumes) the files.
        '''
        return

    def index_filenames(self, *filenames):
        '''Add ``filenames`` to the index

        Convenience method that wraps :meth:`index_files()`

        Params:
            ``filenames``: list of filenames to add to the index.
        '''
        return self.index_files(doc_reader.get_text_files(filenames))

    def index_urls(self, *urls):
        '''Add ``urls`` to the index

        Convenience method that wraps :meth:`index_files()`

        Params:
            ``urls``: list of urls of web pages to add to the index.
        '''
        return self.index_files(doc_reader.get_urls(urls))

    def index_string_buffers(self, named_string_buffers):
        '''Add ``named_string_buffers`` to the index

        Params:
            named_string_buffers: iterable of (name, string) tuples, where
                                  the string contains the data to index.

        '''
        named_files = []
        for (name, string_buffer) in named_string_buffers:
            if isinstance(string_buffer, str):
                string_buffer = unicode(string_buffer)
            named_files.append((name, io.StringIO(string_buffer)))
        self.index_files(named_files)

    @abc.abstractmethod
    def del_docids(self, *docids):
        '''Deletes documents corresponding to docids from the index'''
        return

    @abc.abstractmethod
    def docid_to_name(self, docid):
        '''Returns document name for a given docid'''
        return

    @abc.abstractmethod
    def name_to_docid(self, name):
        '''Returns docid for a given document name'''
        return

    @abc.abstractmethod
    def postings_list(self, term):
        '''
        Return list of (docid, frequency) tuples for docs that contain term
        '''
        return

    def docids_with_terms(self, terms):
        '''Returns a list of docids of docs containing all terms'''
        docs = None  # will hold a set of matching docids
        for term in terms:
            if docs is None:
                docs = set((x[0] for x in self.postings_list(term)))
            else:
                docs.intersection_update(
                    (x[0] for x in self.postings_list(term)))

        # return sorted list
        if docs is None: docs = []
        return sorted(docs)

    def docnames_with_terms(self, *terms):
        '''Returns an iterable of docnames containing terms'''
        if self.config('lowercase'):
            terms = [term.lower() for term in terms]
        return (self.docid_to_name(docid) for docid in self.docids_with_terms(terms))

    def query(self, q):
        '''Finds documents similar to q.

        Params:
            query: the query given as either a string or query vector

        Returns:
            A iterable of (docname, score) tuples sorted by score
        '''
        if isinstance(q, basestring):
            if isinstance(q, str):
                q = unicode(q)
            return self._query(
                term_vec.term_vec(q,
                                  stoplist=self.config('stoplist'),
                                  lowercase=self.config('lowercase')))
        else:
            return self._query(q)

    @abc.abstractmethod
    def _query(self, query_vec):
        '''Finds documents similar to query_vec

        Params:
            query_vec: term vector representing query document

        Returns:
            A iterable of (docname, score) tuples sorted by score
        '''
        return

<|endoftext|>


<commit_before>"""
pysteps.extrapolation.interface
===============================

The functions in the extrapolation module implement the following interface::

    extrapolate(extrap, precip, velocity, num_timesteps,
                outval=np.nan, **keywords)

where *extrap* is an extrapolator object returned by the initialize function,
*precip* is a (m,n) array with input precipitation field to be advected and
*velocity* is a (2,m,n) array containing  the x- and y-components of
the m x n advection field.
num_timesteps is an integer specifying the number of time steps to extrapolate.
The optional argument *outval* specifies the value for pixels advected
from outside the domain.
Optional keyword arguments that are specific to a given extrapolation
method are passed as a dictionary.

The output of each method is an array R_e that includes the time series of
extrapolated fields of shape (num_timesteps, m, n).

.. currentmodule:: pysteps.extrapolation.interface

.. autosummary::
    :toctree: ../generated/

    get_method
    eulerian_persistence
"""

import numpy as np

from pysteps.extrapolation import semilagrangian

def eulerian_persistence(precip, velocity, num_timesteps, outval=np.nan,
                         **kwargs):
    """A dummy extrapolation method to apply Eulerian persistence to a
    two-dimensional precipitation field. The method returns the a sequence
    of the same initial field with no extrapolation applied (i.e. Eulerian
    persistence).

    Parameters
    ----------
    precip : array-like
        Array of shape (m,n) containing the input precipitation field. All
        values are required to be finite.
    velocity : array-like
        Not used by the method.
    num_timesteps : int
        Number of time steps.
    outval : float, optional
        Not used by the method.

    Other Parameters
    ----------------

    return_displacement : bool
        If True, return the total advection velocity (displacement) between the
        initial input field and the advected one integrated along
        the trajectory. Default : False

    Returns
    -------
    out : array or tuple
        If return_displacement=False, return a sequence of the same initial field
        of shape (num_timesteps,m,n). Otherwise, return a tuple containing the
        replicated fields and a (2,m,n) array of zeros.

    References
    ----------
    :cite:`GZ2002` Germann et al (2002)

    """
    del velocity, outval  # Unused by _eulerian_persistence
    return_displacement = kwargs.get("return_displacement", False)

    extrapolated_precip = np.repeat(precip[np.newaxis, :, :, ],
                                    num_timesteps,
                                    axis=0)

    if not return_displacement:
        return extrapolated_precip
    else:
        return extrapolated_precip, np.zeros((2,) + extrapolated_precip.shape)

def _do_nothing(precip, velocity, num_timesteps, outval=np.nan,
                **kwargs):
    """Return None."""
    del precip, velocity, num_timesteps, outval, kwargs  # Unused
    return None

def _return_none(**kwargs):
    del kwargs  # Not used
    return None

_extrapolation_methods = dict()
_extrapolation_methods['eulerian'] = eulerian_persistence
_extrapolation_methods['semilagrangian'] = semilagrangian.extrapolate
_extrapolation_methods[None] = _do_nothing
_extrapolation_methods["none"] = _do_nothing

def get_method(name):
    """Return two-element tuple for the extrapolation method corresponding to
    the given name. The elements of the tuple are callable functions for the
    initializer of the extrapolator and the extrapolation method, respectively.
    The available options are:\n

    +-----------------+--------------------------------------------------------+
    |     Name        |              Description                               |
    +=================+========================================================+
    |  None           | returns None                                           |
    +-----------------+--------------------------------------------------------+
    |  eulerian       | this methods does not apply any advection to the input |
    |                 | precipitation field (Eulerian persistence)             |
    +-----------------+--------------------------------------------------------+
    | semilagrangian  | implementation of the semi-Lagrangian method of        |
    |                 | Germann et al. (2002) :cite:`GZ2002`                   |
    +-----------------+--------------------------------------------------------+

    """
    if isinstance(name, str):
        name = name.lower()

    try:
        return _extrapolation_methods[name]

    except KeyError:
        raise ValueError("Unknown method {}\n".format(name)
                         + "The available methods are:"
                         + str(list(_extrapolation_methods.keys()))) from None
<commit_msg>Update the interface docstring<commit_after>"""
pysteps.extrapolation.interface
===============================

The functions in the extrapolation module implement the following interface::

    extrapolate(extrap, precip, velocity, timesteps, outval=np.nan, **keywords)

where *extrap* is an extrapolator object returned by the initialize function,
*precip* is a (m,n) array with input precipitation field to be advected and
*velocity* is a (2,m,n) array containing  the x- and y-components of
the m x n advection field.
timesteps is an integer or list specifying the time steps to extrapolate. If
an integer is given, a range of uniformly spaced steps 1,2,...,timesteps is
created. If a list is given, it is assumed to represent a sequence of
monotonously increasing time steps. One time unit is assumed to represent the
time step of the advection field.
The optional argument *outval* specifies the value for pixels advected
from outside the domain.
Optional keyword arguments that are specific to a given extrapolation
method are passed as a dictionary.

The output of each method is an array R_e that includes the time series of
extrapolated fields of shape (num_timesteps, m, n).

.. currentmodule:: pysteps.extrapolation.interface

.. autosummary::
    :toctree: ../generated/

    get_method
    eulerian_persistence
"""

import numpy as np

from pysteps.extrapolation import semilagrangian

def eulerian_persistence(precip, velocity, num_timesteps, outval=np.nan,
                         **kwargs):
    """A dummy extrapolation method to apply Eulerian persistence to a
    two-dimensional precipitation field. The method returns the a sequence
    of the same initial field with no extrapolation applied (i.e. Eulerian
    persistence).

    Parameters
    ----------
    precip : array-like
        Array of shape (m,n) containing the input precipitation field. All
        values are required to be finite.
    velocity : array-like
        Not used by the method.
    num_timesteps : int
        Number of time steps.
    outval : float, optional
        Not used by the method.

    Other Parameters
    ----------------

    return_displacement : bool
        If True, return the total advection velocity (displacement) between the
        initial input field and the advected one integrated along
        the trajectory. Default : False

    Returns
    -------
    out : array or tuple
        If return_displacement=False, return a sequence of the same initial field
        of shape (num_timesteps,m,n). Otherwise, return a tuple containing the
        replicated fields and a (2,m,n) array of zeros.

    References
    ----------
    :cite:`GZ2002` Germann et al (2002)

    """
    del velocity, outval  # Unused by _eulerian_persistence
    return_displacement = kwargs.get("return_displacement", False)

    extrapolated_precip = np.repeat(precip[np.newaxis, :, :, ],
                                    num_timesteps,
                                    axis=0)

    if not return_displacement:
        return extrapolated_precip
    else:
        return extrapolated_precip, np.zeros((2,) + extrapolated_precip.shape)

def _do_nothing(precip, velocity, num_timesteps, outval=np.nan,
                **kwargs):
    """Return None."""
    del precip, velocity, num_timesteps, outval, kwargs  # Unused
    return None

def _return_none(**kwargs):
    del kwargs  # Not used
    return None

_extrapolation_methods = dict()
_extrapolation_methods['eulerian'] = eulerian_persistence
_extrapolation_methods['semilagrangian'] = semilagrangian.extrapolate
_extrapolation_methods[None] = _do_nothing
_extrapolation_methods["none"] = _do_nothing

def get_method(name):
    """Return two-element tuple for the extrapolation method corresponding to
    the given name. The elements of the tuple are callable functions for the
    initializer of the extrapolator and the extrapolation method, respectively.
    The available options are:\n

    +-----------------+--------------------------------------------------------+
    |     Name        |              Description                               |
    +=================+========================================================+
    |  None           | returns None                                           |
    +-----------------+--------------------------------------------------------+
    |  eulerian       | this methods does not apply any advection to the input |
    |                 | precipitation field (Eulerian persistence)             |
    +-----------------+--------------------------------------------------------+
    | semilagrangian  | implementation of the semi-Lagrangian method of        |
    |                 | Germann et al. (2002) :cite:`GZ2002`                   |
    +-----------------+--------------------------------------------------------+

    """
    if isinstance(name, str):
        name = name.lower()

    try:
        return _extrapolation_methods[name]

    except KeyError:
        raise ValueError("Unknown method {}\n".format(name)
                         + "The available methods are:"
                         + str(list(_extrapolation_methods.keys()))) from None
<|endoftext|>


<commit_before># -- start license --
# Highly Optimized Object-oriented Many-particle Dynamics -- Blue Edition
# (HOOMD-blue) Open Source Software License Copyright 2009-2015 The Regents of
# the University of Michigan All rights reserved.

# HOOMD-blue may contain modifications ("Contributions") provided, and to which
# copyright is held, by various Contributors who have granted The Regents of the
# University of Michigan the right to modify and/or distribute such Contributions.

# You may redistribute, use, and create derivate works of HOOMD-blue, in source
# and binary forms, provided you abide by the following conditions:

# * Redistributions of source code must retain the above copyright notice, this
# list of conditions, and the following disclaimer both in the code and
# prominently in any materials provided with the distribution.

# * Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions, and the following disclaimer in the documentation and/or
# other materials provided with the distribution.

# * All publications and presentations based on HOOMD-blue, including any reports
# or published results obtained, in whole or in part, with HOOMD-blue, will
# acknowledge its use according to the terms posted at the time of submission on:
# http://codeblue.umich.edu/hoomd-blue/citations.html

# * Any electronic documents citing HOOMD-Blue will link to the HOOMD-Blue website:
# http://codeblue.umich.edu/hoomd-blue/

# * Apart from the above required attributions, neither the name of the copyright
# holder nor the names of HOOMD-blue's contributors may be used to endorse or
# promote products derived from this software without specific prior written
# permission.

# Disclaimer

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND/OR ANY
# WARRANTIES THAT THIS SOFTWARE IS FREE OF INFRINGEMENT ARE DISCLAIMED.

# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# -- end license --

# Maintainer: jglaser / All Developers are free to add commands for new features

## \package hoomd_script.comm
# \brief Commands to support MPI communication

import hoomd;
from hoomd_script import util;
from hoomd_script import globals;
import hoomd_script;

import sys;

## Get the number of ranks
# \returns the number of MPI ranks in this partition
# context.initialize() must be called before get_num_ranks()
# \note Returns 1 in non-mpi builds
def get_num_ranks():
    hoomd_script.context._verify_init();
    if hoomd.is_MPI_available():
        return globals.exec_conf.getNRanks();
    else:
        return 1;

## Return the current rank
# context.initialize() must be called before get_rank()
# \note Always returns 0 in non-mpi builds
def get_rank():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        return globals.exec_conf.getRank()
    else:
        return 0;

## Return the current partition
# context.initialize() must be called before get_partition()
# \note Always returns 0 in non-mpi builds
def get_partition():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        return globals.exec_conf.getPartition()
    else:
        return 0;

## Perform a MPI barrier synchronization inside a partition
# \note does nothing in in non-MPI builds
def barrier_all():
    if hoomd.is_MPI_available():
        hoomd.mpi_barrier_world();

## Perform a MPI barrier synchronization inside a partition
# context.initialize() must be called before barrier()
# \note does nothing in in non-MPI builds
def barrier():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        globals.exec_conf.barrier()

## Balances the domain decomposition
#
# A single domain %decomposition is defined for the simulation.
# A standard domain %decomposition divides the simulation box into equal volumes along the Cartesian axes while minimizing
# the surface area between domains. This works well for systems where particles are uniformly distributed and
# there is equal computational load for each domain, and is the default behavior in HOOMD-blue. If no %decomposition is
# specified for an MPI run, a uniform %decomposition is automatically constructed on initialization.
#
# In simulations with density gradients, such as a vapor-liquid interface, there can be a considerable imbalance of
# particles between different ranks. The simulation time then becomes limited by the slowest processor. It may then be
# advantageous in certain systems to create domains of unequal volume, for example, by increasing the volume of less
# dense regions of the simulation box in order to balance the number of particles.
#
# The %decomposition command allows the user to control the geometry and positions of the %decomposition.
# The fractional width of the first \f$n_i - 1\f$ domains is specified along each dimension, where
# \f$n_i\f$ is the number of ranks desired along dimension \f$i\f$. If no cut planes are specified, then a uniform
# spacing is assumed. The number of domains with uniform spacing can also be specified. If the desired %decomposition
# is not commensurate with the number of ranks available (for example, a 3x3x3 decomposition when only 8 ranks are
# available), then a default uniform spacing is chosen. For the best control, the user should specify the number of
# ranks in each dimension even if uniform spacing is desired.
#
# \warning The %decomposition command will override specified command line options.
#
class decomposition():
    ## Create a balanced domain decomposition
    # \param x First nx-1 fractional domain widths
    # \param y First ny-1 fractional domain widths
    # \param z First nz-1 fractional domain widths
    # \param nx Number of processors to uniformly space in x dimension (if x list is None)
    # \param ny Number of processors to uniformly space in y dimension (if y list is None)
    # \param nz Number of processors to uniformly space in z dimension (if z list is None)
    #
    # Priority is always given to specified arguments over the command line arguments. If one of these is not set but
    # a command line option is, then the command line option is used. Otherwise, a default %decomposition is chosen.
    #
    # \b Examples:
    # \code
    # comm.decomposition(x=0.4, ny=2, nz=2)
    # comm.decomposition(nx=2, y=0.8, z=[0.2,0.3])
    # \endcode
    #
    # \note This command must be invoked *before* the system is initialized because particles are decomposed at this time.
    #
    # \note The domain size cannot be chosen arbitrarily small. There are restrictions placed on the %decomposition by the
    #       ghost layer width set by the %pair potentials. An error will be raised at run time if the ghost layer width
    #       exceeds half the shortest domain size.
    #
    # \warning Both fractional widths and the number of processors cannot be set simultaneously, and an error will be
    #          raised if both are set.
    def __init__(self, x=None, y=None, z=None, nx=None, ny=None, nz=None):
        util.print_status_line()

        # check that system is not initialized
        if globals.system is not None:
            globals.msg.error("comm.decomposition: cannot modify decomposition after system is initialized. Call before init.*\n")
            raise RuntimeError("Cannot create decomposition after system is initialized. Call before init.*")

        # check that the context has been initialized though
        if globals.exec_conf is None:
            globals.msg.error("comm.decomposition: call context.initialize() before decomposition can be set\n")
            raise RuntimeError("Cannot initialize decomposition without context.initialize() first")

        # check that there are ranks available for decomposition
        if globals.exec_conf.getNRanks() == 1:
            globals.msg.warning("Only 1 rank in system, ignoring decomposition to use optimized code pathways.\n")
            return
        else:
            self.x = []
            self.y = []
            self.z = []
            self.nx = 0
            self.ny = 0
            self.nz = 0
            self.uniform_x = True
            self.uniform_y = True
            self.uniform_z = True

            util._disable_status_lines = True
            self.set_params(x,y,z,nx,ny,nz)
            util._disable_status_lines = False

            # do a one time update of the cuts to the global values if a global is set
            if not self.x and self.nx == 0 and globals.options.nx is not None:
                self.nx = globals.options.nx
                self.uniform_x = True
            if not self.y and self.ny == 0 and globals.options.ny is not None:
                self.ny = globals.options.ny
                self.uniform_y = True
            if not self.z and self.nz == 0:
                if globals.options.linear is True:
                    self.nz = globals.exec_conf.getNRanks()
                    self.uniform_z = True
                elif globals.options.nz is not None:
                    self.nz = globals.options.nz
                    self.uniform_z = True

            # set the global decomposition to this class
            if globals.decomposition is not None:
                globals.msg.warning("comm.decomposition: overriding currently defined domain decomposition\n")

            globals.decomposition = self

    ## Set parameters for the decomposition before initialization.
    # \param x First nx-1 fractional domain widths
    # \param y First ny-1 fractional domain widths
    # \param z First nz-1 fractional domain widths
    # \param nx Number of processors to uniformly space in x dimension (if x list is None)
    # \param ny Number of processors to uniformly space in y dimension (if y list is None)
    # \param nz Number of processors to uniformly space in z dimension (if z list is None)
    #
    # \b Examples:
    # \code
    # decomposition.set_params(x=[0.2])
    # decomposition.set_params(nx=1, y=[0.3,0.4], nz=2)
    # \endcode
    def set_params(self,x=None,y=None,z=None,nx=None,ny=None,nz=None):
        util.print_status_line()

        if (x is not None and nx is not None) or (y is not None and ny is not None) or (z is not None and nz is not None):
            globals.msg.error("comm.decomposition: cannot set fractions and number of processors simultaneously\n")
            raise RuntimeError("Cannot set fractions and number of processors simultaneously")

        # if x is set, use it. otherwise, if nx is set, compute x and set it
        if x is not None:
            # recast single floats as lists that can be iterated, this is the only single input we should expect
            if isinstance(x, float):
                self.x = [x]
            else:
                self.x = x
            self.uniform_x = False
        elif nx is not None:
            self.nx = nx
            self.uniform_x = True

        # do the same in y
        if y is not None:
            if isinstance(y, float):
                self.y = [y]
            else:
                self.y = y
            self.uniform_y = False
        elif ny is not None:
            self.ny = ny
            self.uniform_y = True

        # do the same in z (but also use the linear command line option if it is present, which supersedes nz)
        if z is not None:
            if isinstance(z, float):
                self.z = [z]
            else:
                self.z = z
            self.uniform_z = False
        elif nz is not None:
            self.nz = nz
            self.uniform_z = True

    ## \internal
    # \brief Delayed construction of the C++ object for this balanced decomposition
    # \param box Global simulation box for decomposition
    def _make_cpp_decomposition(self, box):
        # if the box is uniform in all directions, just use these values
        if self.uniform_x and self.uniform_y and self.uniform_z:
            self.cpp_dd = hoomd.DomainDecomposition(globals.exec_conf, box.getL(), self.nx, self.ny, self.nz, not globals.options.onelevel)
            return self.cpp_dd

        # otherwise, make the fractional decomposition
        try:
            fxs = hoomd.std_vector_scalar()
            fys = hoomd.std_vector_scalar()
            fzs = hoomd.std_vector_scalar()

            # if uniform, correct the fractions to be uniform as well
            if self.uniform_x and self.nx > 0:
                self.x = [1.0/self.nx]*(self.nx-1)
            if self.uniform_y and self.ny > 0:
                self.y = [1.0/self.ny]*(self.ny-1)
            if self.uniform_z and self.nz > 0:
                self.z = [1.0/self.nz]*(self.nz-1)

            if get_rank() == 0:
                print self.x
                print self.y
                print self.z

            sum_x = sum_y = sum_z = 0.0
            tol = 1.0e-5
            for i in self.x:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fxs.append(i)
                sum_x += i
            if sum_x >= 1.0 - tol or sum_x <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in x must lie between 0.0 and 1.0")

            for i in self.y:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fys.append(i)
                sum_y += i
            if sum_y >= 1.0 - tol or sum_y <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in y must lie between 0.0 and 1.0")

            for i in self.z:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fzs.append(i)
                sum_z += i
            if sum_z >= 1.0 - tol or sum_z <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in z must lie between 0.0 and 1.0")

            self.cpp_dd = hoomd.DomainDecomposition(globals.exec_conf, box.getL(), fxs, fys, fzs)
            return self.cpp_dd

        except TypeError,te:
            globals.msg.error("Fractional cuts must be iterable (list, tuple, etc.)\n")
            raise te
<commit_msg>Remove stray print statements<commit_after># -- start license --
# Highly Optimized Object-oriented Many-particle Dynamics -- Blue Edition
# (HOOMD-blue) Open Source Software License Copyright 2009-2015 The Regents of
# the University of Michigan All rights reserved.

# HOOMD-blue may contain modifications ("Contributions") provided, and to which
# copyright is held, by various Contributors who have granted The Regents of the
# University of Michigan the right to modify and/or distribute such Contributions.

# You may redistribute, use, and create derivate works of HOOMD-blue, in source
# and binary forms, provided you abide by the following conditions:

# * Redistributions of source code must retain the above copyright notice, this
# list of conditions, and the following disclaimer both in the code and
# prominently in any materials provided with the distribution.

# * Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions, and the following disclaimer in the documentation and/or
# other materials provided with the distribution.

# * All publications and presentations based on HOOMD-blue, including any reports
# or published results obtained, in whole or in part, with HOOMD-blue, will
# acknowledge its use according to the terms posted at the time of submission on:
# http://codeblue.umich.edu/hoomd-blue/citations.html

# * Any electronic documents citing HOOMD-Blue will link to the HOOMD-Blue website:
# http://codeblue.umich.edu/hoomd-blue/

# * Apart from the above required attributions, neither the name of the copyright
# holder nor the names of HOOMD-blue's contributors may be used to endorse or
# promote products derived from this software without specific prior written
# permission.

# Disclaimer

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND/OR ANY
# WARRANTIES THAT THIS SOFTWARE IS FREE OF INFRINGEMENT ARE DISCLAIMED.

# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# -- end license --

# Maintainer: jglaser / All Developers are free to add commands for new features

## \package hoomd_script.comm
# \brief Commands to support MPI communication

import hoomd;
from hoomd_script import util;
from hoomd_script import globals;
import hoomd_script;

import sys;

## Get the number of ranks
# \returns the number of MPI ranks in this partition
# context.initialize() must be called before get_num_ranks()
# \note Returns 1 in non-mpi builds
def get_num_ranks():
    hoomd_script.context._verify_init();
    if hoomd.is_MPI_available():
        return globals.exec_conf.getNRanks();
    else:
        return 1;

## Return the current rank
# context.initialize() must be called before get_rank()
# \note Always returns 0 in non-mpi builds
def get_rank():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        return globals.exec_conf.getRank()
    else:
        return 0;

## Return the current partition
# context.initialize() must be called before get_partition()
# \note Always returns 0 in non-mpi builds
def get_partition():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        return globals.exec_conf.getPartition()
    else:
        return 0;

## Perform a MPI barrier synchronization inside a partition
# \note does nothing in in non-MPI builds
def barrier_all():
    if hoomd.is_MPI_available():
        hoomd.mpi_barrier_world();

## Perform a MPI barrier synchronization inside a partition
# context.initialize() must be called before barrier()
# \note does nothing in in non-MPI builds
def barrier():
    hoomd_script.context._verify_init();

    if hoomd.is_MPI_available():
        globals.exec_conf.barrier()

## Balances the domain decomposition
#
# A single domain %decomposition is defined for the simulation.
# A standard domain %decomposition divides the simulation box into equal volumes along the Cartesian axes while minimizing
# the surface area between domains. This works well for systems where particles are uniformly distributed and
# there is equal computational load for each domain, and is the default behavior in HOOMD-blue. If no %decomposition is
# specified for an MPI run, a uniform %decomposition is automatically constructed on initialization.
#
# In simulations with density gradients, such as a vapor-liquid interface, there can be a considerable imbalance of
# particles between different ranks. The simulation time then becomes limited by the slowest processor. It may then be
# advantageous in certain systems to create domains of unequal volume, for example, by increasing the volume of less
# dense regions of the simulation box in order to balance the number of particles.
#
# The %decomposition command allows the user to control the geometry and positions of the %decomposition.
# The fractional width of the first \f$n_i - 1\f$ domains is specified along each dimension, where
# \f$n_i\f$ is the number of ranks desired along dimension \f$i\f$. If no cut planes are specified, then a uniform
# spacing is assumed. The number of domains with uniform spacing can also be specified. If the desired %decomposition
# is not commensurate with the number of ranks available (for example, a 3x3x3 decomposition when only 8 ranks are
# available), then a default uniform spacing is chosen. For the best control, the user should specify the number of
# ranks in each dimension even if uniform spacing is desired.
#
# \warning The %decomposition command will override specified command line options.
#
class decomposition():
    ## Create a balanced domain decomposition
    # \param x First nx-1 fractional domain widths
    # \param y First ny-1 fractional domain widths
    # \param z First nz-1 fractional domain widths
    # \param nx Number of processors to uniformly space in x dimension (if x list is None)
    # \param ny Number of processors to uniformly space in y dimension (if y list is None)
    # \param nz Number of processors to uniformly space in z dimension (if z list is None)
    #
    # Priority is always given to specified arguments over the command line arguments. If one of these is not set but
    # a command line option is, then the command line option is used. Otherwise, a default %decomposition is chosen.
    #
    # \b Examples:
    # \code
    # comm.decomposition(x=0.4, ny=2, nz=2)
    # comm.decomposition(nx=2, y=0.8, z=[0.2,0.3])
    # \endcode
    #
    # \note This command must be invoked *before* the system is initialized because particles are decomposed at this time.
    #
    # \note The domain size cannot be chosen arbitrarily small. There are restrictions placed on the %decomposition by the
    #       ghost layer width set by the %pair potentials. An error will be raised at run time if the ghost layer width
    #       exceeds half the shortest domain size.
    #
    # \warning Both fractional widths and the number of processors cannot be set simultaneously, and an error will be
    #          raised if both are set.
    def __init__(self, x=None, y=None, z=None, nx=None, ny=None, nz=None):
        util.print_status_line()

        # check that system is not initialized
        if globals.system is not None:
            globals.msg.error("comm.decomposition: cannot modify decomposition after system is initialized. Call before init.*\n")
            raise RuntimeError("Cannot create decomposition after system is initialized. Call before init.*")

        # check that the context has been initialized though
        if globals.exec_conf is None:
            globals.msg.error("comm.decomposition: call context.initialize() before decomposition can be set\n")
            raise RuntimeError("Cannot initialize decomposition without context.initialize() first")

        # check that there are ranks available for decomposition
        if globals.exec_conf.getNRanks() == 1:
            globals.msg.warning("Only 1 rank in system, ignoring decomposition to use optimized code pathways.\n")
            return
        else:
            self.x = []
            self.y = []
            self.z = []
            self.nx = 0
            self.ny = 0
            self.nz = 0
            self.uniform_x = True
            self.uniform_y = True
            self.uniform_z = True

            util._disable_status_lines = True
            self.set_params(x,y,z,nx,ny,nz)
            util._disable_status_lines = False

            # do a one time update of the cuts to the global values if a global is set
            if not self.x and self.nx == 0 and globals.options.nx is not None:
                self.nx = globals.options.nx
                self.uniform_x = True
            if not self.y and self.ny == 0 and globals.options.ny is not None:
                self.ny = globals.options.ny
                self.uniform_y = True
            if not self.z and self.nz == 0:
                if globals.options.linear is True:
                    self.nz = globals.exec_conf.getNRanks()
                    self.uniform_z = True
                elif globals.options.nz is not None:
                    self.nz = globals.options.nz
                    self.uniform_z = True

            # set the global decomposition to this class
            if globals.decomposition is not None:
                globals.msg.warning("comm.decomposition: overriding currently defined domain decomposition\n")

            globals.decomposition = self

    ## Set parameters for the decomposition before initialization.
    # \param x First nx-1 fractional domain widths
    # \param y First ny-1 fractional domain widths
    # \param z First nz-1 fractional domain widths
    # \param nx Number of processors to uniformly space in x dimension (if x list is None)
    # \param ny Number of processors to uniformly space in y dimension (if y list is None)
    # \param nz Number of processors to uniformly space in z dimension (if z list is None)
    #
    # \b Examples:
    # \code
    # decomposition.set_params(x=[0.2])
    # decomposition.set_params(nx=1, y=[0.3,0.4], nz=2)
    # \endcode
    def set_params(self,x=None,y=None,z=None,nx=None,ny=None,nz=None):
        util.print_status_line()

        if (x is not None and nx is not None) or (y is not None and ny is not None) or (z is not None and nz is not None):
            globals.msg.error("comm.decomposition: cannot set fractions and number of processors simultaneously\n")
            raise RuntimeError("Cannot set fractions and number of processors simultaneously")

        # if x is set, use it. otherwise, if nx is set, compute x and set it
        if x is not None:
            # recast single floats as lists that can be iterated, this is the only single input we should expect
            if isinstance(x, float):
                self.x = [x]
            else:
                self.x = x
            self.uniform_x = False
        elif nx is not None:
            self.nx = nx
            self.uniform_x = True

        # do the same in y
        if y is not None:
            if isinstance(y, float):
                self.y = [y]
            else:
                self.y = y
            self.uniform_y = False
        elif ny is not None:
            self.ny = ny
            self.uniform_y = True

        # do the same in z (but also use the linear command line option if it is present, which supersedes nz)
        if z is not None:
            if isinstance(z, float):
                self.z = [z]
            else:
                self.z = z
            self.uniform_z = False
        elif nz is not None:
            self.nz = nz
            self.uniform_z = True

    ## \internal
    # \brief Delayed construction of the C++ object for this balanced decomposition
    # \param box Global simulation box for decomposition
    def _make_cpp_decomposition(self, box):
        # if the box is uniform in all directions, just use these values
        if self.uniform_x and self.uniform_y and self.uniform_z:
            self.cpp_dd = hoomd.DomainDecomposition(globals.exec_conf, box.getL(), self.nx, self.ny, self.nz, not globals.options.onelevel)
            return self.cpp_dd

        # otherwise, make the fractional decomposition
        try:
            fxs = hoomd.std_vector_scalar()
            fys = hoomd.std_vector_scalar()
            fzs = hoomd.std_vector_scalar()

            # if uniform, correct the fractions to be uniform as well
            if self.uniform_x and self.nx > 0:
                self.x = [1.0/self.nx]*(self.nx-1)
            if self.uniform_y and self.ny > 0:
                self.y = [1.0/self.ny]*(self.ny-1)
            if self.uniform_z and self.nz > 0:
                self.z = [1.0/self.nz]*(self.nz-1)

            sum_x = sum_y = sum_z = 0.0
            tol = 1.0e-5
            for i in self.x:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fxs.append(i)
                sum_x += i
            if sum_x >= 1.0 - tol or sum_x <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in x must lie between 0.0 and 1.0")

            for i in self.y:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fys.append(i)
                sum_y += i
            if sum_y >= 1.0 - tol or sum_y <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in y must lie between 0.0 and 1.0")

            for i in self.z:
                if i <= -tol or i >= 1.0 - tol:
                    globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                    raise RuntimeError("Fractional decomposition must be between 0.0 and 1.0")
                fzs.append(i)
                sum_z += i
            if sum_z >= 1.0 - tol or sum_z <= -tol:
                globals.msg.error("comm.decomposition: fraction must be between 0.0 and 1.0\n")
                raise RuntimeError("Sum of decomposition in z must lie between 0.0 and 1.0")

            self.cpp_dd = hoomd.DomainDecomposition(globals.exec_conf, box.getL(), fxs, fys, fzs)
            return self.cpp_dd

        except TypeError,te:
            globals.msg.error("Fractional cuts must be iterable (list, tuple, etc.)\n")
            raise te
<|endoftext|>


<commit_before>import numpy as np
import skimage.io
import skimage.draw
import skimage.measure
import tempfile

import pymorse

from datetime import datetime
from typing import Tuple, Optional

from fire_rs.geodata.geo_data import GeoData

class MorseWildfire:
    """Communicate with Morse to display a fire in the scene"""

    def __init__(self, address: 'Tuple[str, int]', terrain_object: 'str'):
        self.address = address
        self.terrain_object = terrain_object

        self.fire_map = None
        self.layer = None

        self.fire_image = None
        self.color = (255, 255, 255)

        self.morse_conn = None  # type: Optional[pymorse.Morse]
        self._morse_timeout = 5.

    def close(self, *args):
        """Close Morse connection"""
        if self.morse_conn is not None:
            self.morse_conn.close(*args)
            self.morse_conn = None

    def set_wildfire_prediction_map(self, fire_map: GeoData, layer='ignition'):
        """Set the wildfire ignition time map"""
        self.fire_map = fire_map
        self.layer = layer

    def _update_fire_image(self, time: 'float'):
        fire_image = np.zeros((*self.fire_map.data.shape, 3), dtype=np.uint8)
        # Create contour with scikit-image
        contours = skimage.measure.find_contours(self.fire_map.data[self.layer], time)
        # Print contour as binary image
        for contour in contours:
            for pt_i in range(len(contour)):
                rr, cc = skimage.draw.line(*np.asarray(contour[pt_i - 1], dtype=int),
                                           *np.asarray(contour[pt_i], dtype=int))
                fire_image[rr, cc] = self.color

        self.fire_image = fire_image

    def update(self, time: 'Optional[float]' = None):
        """Update wildfire texture at current time in morse"""
        if self.fire_map is None:
            raise ValueError("The wildfire ignition time map is not set")

        if time is None:
            time = datetime.now().timestamp()

        self._update_fire_image(time)

        with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as t_file:
            skimage.io.imsave(t_file, self.fire_image)

            if self.morse_conn is None or not self.morse_conn.is_up():
                self.morse_conn = pymorse.Morse(*self.address)

            self.morse_conn.rpc_t(self._morse_timeout, "simulation", "set_texture",
                                  str(self.terrain_object), str(t_file.name))

    def __enter__(self):
        if self.morse_conn is None or not self.morse_conn.is_up():
            self.morse_conn = pymorse.Morse(*self.address)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not exc_type:
            self.close()
        else:
            self.close(True)
            return False  # re-raise exception
<commit_msg>Fix orientation of the wildfire texture sent to morse<commit_after>import numpy as np
import skimage.io
import skimage.draw
import skimage.measure
import tempfile

import pymorse

from datetime import datetime
from typing import Tuple, Optional

from fire_rs.geodata.geo_data import GeoData

class MorseWildfire:
    """Communicate with Morse to display a fire in the scene"""

    def __init__(self, address: 'Tuple[str, int]', terrain_object: 'str'):
        self.address = address
        self.terrain_object = terrain_object

        self.fire_map = None
        self.layer = None

        self.fire_image = None
        self.color = (255, 255, 255)

        self.morse_conn = None  # type: Optional[pymorse.Morse]
        self._morse_timeout = 5.

    def close(self, *args):
        """Close Morse connection"""
        if self.morse_conn is not None:
            self.morse_conn.close(*args)
            self.morse_conn = None

    def set_wildfire_prediction_map(self, fire_map: GeoData, layer='ignition'):
        """Set the wildfire ignition time map"""
        self.fire_map = fire_map
        self.layer = layer

    def _update_fire_image(self, time: 'float'):
        fire_image = np.zeros((*self.fire_map.data.shape, 3), dtype=np.uint8)
        # Create contour with scikit-image
        contours = skimage.measure.find_contours(self.fire_map.data[self.layer], time)
        # Print contour as binary image
        for contour in contours:
            for pt_i in range(len(contour)):
                rr, cc = skimage.draw.line(*np.asarray(contour[pt_i - 1], dtype=int),
                                           *np.asarray(contour[pt_i], dtype=int))
                fire_image[rr, cc] = self.color

        self.fire_image = fire_image

    def update(self, time: 'Optional[float]' = None):
        """Update wildfire texture at current time in morse"""
        if self.fire_map is None:
            raise ValueError("The wildfire ignition time map is not set")

        if time is None:
            time = datetime.now().timestamp()

        self._update_fire_image(time)

        with tempfile.NamedTemporaryFile(suffix=".png", delete=True) as t_file:
            skimage.io.imsave(t_file, np.transpose(self.fire_image, (1,0,2))[::-1,::1,...])

            if self.morse_conn is None or not self.morse_conn.is_up():
                self.morse_conn = pymorse.Morse(*self.address)

            self.morse_conn.rpc_t(self._morse_timeout, "simulation", "set_texture",
                                  str(self.terrain_object), str(t_file.name))

    def __enter__(self):
        if self.morse_conn is None or not self.morse_conn.is_up():
            self.morse_conn = pymorse.Morse(*self.address)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not exc_type:
            self.close()
        else:
            self.close(True)
            return False  # re-raise exception
<|endoftext|>


<commit_before>__version__ = '1.7.0'
<commit_msg>Python 1.7.1<commit_after>__version__ = '1.7.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
#
# dap.py
#
# Created by José Sánchez-Gallego on 15 Jun 2016.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import re

import astropy.io.fits as fits

import marvin

def maps_file2dict_of_props(maps_file, x, y):
    """From a MAPS file, creates a dictionary of properties for the `(x, y)` spaxel."""

    if not isinstance(maps_file, fits.HDUList):
        fits_map = fits.open(maps_file)
    else:
        fits_map = maps_file

    dict_of_props = {}

    # Selects extensions that contains a "category", i.e., if it has a value,
    # a mask and an ivar.
    valid_extensions = [extension.name for extension in fits_map
                        if extension.name + '_ivar' in fits_map and extension.name + '_mask']

    for ext_name in valid_extensions:
        category = ext_name
        extension = fits_map[ext_name]
        ivar_extension = fits_map[ext_name + '_ivar']
        mask_extension = fits_map[ext_name + '_mask']

        # Checks whether we have channels for this category
        if len(extension.data.shape) == 2:

            dict_of_props[category] = {
                'NA': {'value': extension.data[y, x],
                       'ivar': ivar_extension.data[y, x],
                       'mask': mask_extension.data[y, x]},
                'unit': extension.header['BUNIT'] if 'BUNIT' in extension.header else ''}

        elif len(extension.data.shape) == 3:

            # Gets the channels and creates the names.
            channel_keys = [key for key in extension.header.keys() if re.match('C[0-9]+', key)]
            names = [re.sub('\-+', '-', extension.header[key]) for key in channel_keys]

            dict_of_props[category] = {
                name: {'value': extension.data[ii, y, x],
                       'ivar': ivar_extension.data[ii, y, x],
                       'mask': mask_extension.data[ii, y, x]}
                for ii, name in enumerate(names)}
            dict_of_props[category]['unit'] = (extension.header['BUNIT']
                                               if 'BUNIT' in extension.header else '')

        else:
            raise ValueError('error when obtaining properties from '
                             'extension {0}: it has {1} dimensions!'
                             .format(ext_name, len(extension.data.shape)))

    return dict_of_props

def maps_db2dict_of_props(maps_db_file, x, y):
    """From a ``dapdb.File``, creates a dictionary of properties for the `(x, y)` spaxel."""

    assert isinstance(maps_db_file, marvin.marvindb.dapdb.File)

    dict_of_props = {}

    # Adds the emission lines
    for emline in maps_db_file.emlines:
        value = emline.value[y][x]
        ivar = emline.ivar[y][x] if emline.ivar else 0
        mask = emline.mask[y][x] if emline.mask else 1
        parameter = 'EMLINE_' + emline.parameter.name
        unit = emline.parameter.unit
        channel_name = emline.type.name + '-' + str(emline.type.rest_wavelength)

        if parameter not in dict_of_props:
            dict_of_props[parameter] = {}

        dict_of_props[parameter][channel_name] = {'value': value,
                                                  'ivar': ivar,
                                                  'mask': mask}
        dict_of_props[parameter]['unit'] = unit

    # Idem with the stellar kinematics
    for stellar_kin in maps_db_file.stellarkins:
        value = stellar_kin.value[y][x]
        ivar = stellar_kin.ivar[y][x] if stellar_kin.ivar else 0
        mask = stellar_kin.mask[y][x] if stellar_kin.mask else 1
        parameter = 'STELLAR_' + stellar_kin.parameter.name
        unit = stellar_kin.parameter.unit

        dict_of_props[parameter] = {'NA': {'value': value,
                                           'ivar': ivar,
                                           'mask': mask}}
        dict_of_props[parameter]['unit'] = unit

    # Idem with the spectral indices
    for specindex in maps_db_file.specindices:
        value = specindex.value[y][x]
        ivar = specindex.ivar[y][x] if specindex.ivar else 0
        mask = specindex.mask[y][x] if specindex.mask else 1
        unit = specindex.type.unit
        channel_name = specindex.type.name

        dict_of_props['SPECINDEX'] = {}
        dict_of_props['SPECINDEX'][channel_name] = {'value': value,
                                                    'ivar': ivar,
                                                    'mask': mask}
        dict_of_props['SPECINDEX']['unit'] = unit

    return dict_of_props

def get_dict_of_props_api(plateifu, x, y):
    """Returns a dictionary of DAP properties by performing an API call."""

    url = marvin.config.urlmap['api']['getdap_props']['url']

    try:
        path = 'x={0}/y={1}'.format(x, y)
        url_full = url.format(name=plateifu, path=path)
        response = marvin.api.api.Interaction(url_full)
    except Exception as ee:
        raise marvin.core.exceptions.MarvinError(
            'found a problem when checking if remote maps exists: {0}'.format(str(ee)))

    data = response.getData()

    return data

def list_categories(hdu=None, dapver=marvin.config.dapver):
    """Returns a list of categories for a DAP :class:`~marvin.tools.maps.Maps`.

    Parameters:
        hdu (``astropy.io.fits.HDUList`` or None):
            If ``hdu`` is defined, it will use the extension names in the HDU.
            Otherwise it will use the DB.
        dapver (string or None):
            The DAP version to use.

    Returns:
        list_categories (dict):
            A dictionary in which keys are the valid categories an the values
            are the valid channels or None if channels are not appliable for
            that category.

    """

    cat_dict = {}

    if hdu is not None:
        categories = [ext.name.lower() for ext in hdu
                      if ext.name + '_ivar' in hdu and ext.name + '_mask']

        for category in categories:
            extension = hdu[category]
            if len(extension.data.shape) == 2:
                cat_dict[category] = None

            elif len(extension.data.shape) == 3:

                # Gets the channels and creates the names.
                channel_keys = [key for key in extension.header.keys()
                                if re.match('C[0-9]+', key)]
                names = [re.sub('\-+', '-', extension.header[key])
                         for key in channel_keys]

                cat_dict[category] = [name.lower() for name in names]

        return cat_dict

    else:

        mdb = marvin.marvindb

        # Emission lines
        emline_params = ['emline_' + row.name.lower()
                         for row in mdb.session.query(mdb.dapdb.EmLineParameter).all()]
        emline_channels = ['{0}-{1}'.format(row.name.lower(), row.rest_wavelength)
                           for row in mdb.session.query(mdb.dapdb.EmLineType).all()]
        for param in emline_params:
            cat_dict[param] = emline_channels

        # Stellar kinematics
        stell_kin_params = ['stellar_' + row.name.lower()
                            for row in mdb.session.query(mdb.dapdb.StellarKinParameter).all()]
        for param in stell_kin_params:
            cat_dict[param] = None

        # Spectral indices
        specindex_channels = [row.name.lower()
                              for row in mdb.session.query(mdb.dapdb.SpecIndexType).all()]
        cat_dict['specindex'] = specindex_channels

    return cat_dict
<commit_msg>fixed bug that created the values for only the last spectra index when loaded from the DB<commit_after>#!/usr/bin/env python
# encoding: utf-8
#
# dap.py
#
# Created by José Sánchez-Gallego on 15 Jun 2016.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import re

import astropy.io.fits as fits

import marvin

def maps_file2dict_of_props(maps_file, x, y):
    """From a MAPS file, creates a dictionary of properties for the `(x, y)` spaxel."""

    if not isinstance(maps_file, fits.HDUList):
        fits_map = fits.open(maps_file)
    else:
        fits_map = maps_file

    dict_of_props = {}

    # Selects extensions that contains a "category", i.e., if it has a value,
    # a mask and an ivar.
    valid_extensions = [extension.name for extension in fits_map
                        if extension.name + '_ivar' in fits_map and extension.name + '_mask']

    for ext_name in valid_extensions:
        category = ext_name
        extension = fits_map[ext_name]
        ivar_extension = fits_map[ext_name + '_ivar']
        mask_extension = fits_map[ext_name + '_mask']

        # Checks whether we have channels for this category
        if len(extension.data.shape) == 2:

            dict_of_props[category] = {
                'NA': {'value': extension.data[y, x],
                       'ivar': ivar_extension.data[y, x],
                       'mask': mask_extension.data[y, x]},
                'unit': extension.header['BUNIT'] if 'BUNIT' in extension.header else ''}

        elif len(extension.data.shape) == 3:

            # Gets the channels and creates the names.
            channel_keys = [key for key in extension.header.keys() if re.match('C[0-9]+', key)]
            names = [re.sub('\-+', '-', extension.header[key]) for key in channel_keys]

            dict_of_props[category] = {
                name: {'value': extension.data[ii, y, x],
                       'ivar': ivar_extension.data[ii, y, x],
                       'mask': mask_extension.data[ii, y, x]}
                for ii, name in enumerate(names)}
            dict_of_props[category]['unit'] = (extension.header['BUNIT']
                                               if 'BUNIT' in extension.header else '')

        else:
            raise ValueError('error when obtaining properties from '
                             'extension {0}: it has {1} dimensions!'
                             .format(ext_name, len(extension.data.shape)))

    return dict_of_props

def maps_db2dict_of_props(maps_db_file, x, y):
    """From a ``dapdb.File``, creates a dictionary of properties for the `(x, y)` spaxel."""

    assert isinstance(maps_db_file, marvin.marvindb.dapdb.File)

    dict_of_props = {}

    # Adds the emission lines
    for emline in maps_db_file.emlines:
        value = emline.value[y][x]
        ivar = emline.ivar[y][x] if emline.ivar else 0
        mask = emline.mask[y][x] if emline.mask else 1
        parameter = 'EMLINE_' + emline.parameter.name
        unit = emline.parameter.unit
        channel_name = emline.type.name + '-' + str(emline.type.rest_wavelength)

        if parameter not in dict_of_props:
            dict_of_props[parameter] = {}

        dict_of_props[parameter][channel_name] = {'value': value,
                                                  'ivar': ivar,
                                                  'mask': mask}
        dict_of_props[parameter]['unit'] = unit

    # Idem with the stellar kinematics
    for stellar_kin in maps_db_file.stellarkins:
        value = stellar_kin.value[y][x]
        ivar = stellar_kin.ivar[y][x] if stellar_kin.ivar else 0
        mask = stellar_kin.mask[y][x] if stellar_kin.mask else 1
        parameter = 'STELLAR_' + stellar_kin.parameter.name
        unit = stellar_kin.parameter.unit

        dict_of_props[parameter] = {'NA': {'value': value,
                                           'ivar': ivar,
                                           'mask': mask}}
        dict_of_props[parameter]['unit'] = unit

    # Idem with the spectral indices
    dict_of_props['SPECINDEX'] = {}
    for specindex in maps_db_file.specindices:
        value = specindex.value[y][x]
        ivar = specindex.ivar[y][x] if specindex.ivar else 0
        mask = specindex.mask[y][x] if specindex.mask else 1
        unit = specindex.type.unit
        channel_name = specindex.type.name

        dict_of_props['SPECINDEX'][channel_name] = {'value': value,
                                                    'ivar': ivar,
                                                    'mask': mask}
        dict_of_props['SPECINDEX']['unit'] = unit

    return dict_of_props

def get_dict_of_props_api(plateifu, x, y):
    """Returns a dictionary of DAP properties by performing an API call."""

    url = marvin.config.urlmap['api']['getdap_props']['url']

    try:
        path = 'x={0}/y={1}'.format(x, y)
        url_full = url.format(name=plateifu, path=path)
        response = marvin.api.api.Interaction(url_full)
    except Exception as ee:
        raise marvin.core.exceptions.MarvinError(
            'found a problem when checking if remote maps exists: {0}'.format(str(ee)))

    data = response.getData()

    return data

def list_categories(hdu=None, dapver=marvin.config.dapver):
    """Returns a list of categories for a DAP :class:`~marvin.tools.maps.Maps`.

    Parameters:
        hdu (``astropy.io.fits.HDUList`` or None):
            If ``hdu`` is defined, it will use the extension names in the HDU.
            Otherwise it will use the DB.
        dapver (string or None):
            The DAP version to use.

    Returns:
        list_categories (dict):
            A dictionary in which keys are the valid categories an the values
            are the valid channels or None if channels are not appliable for
            that category.

    """

    cat_dict = {}

    if hdu is not None:
        categories = [ext.name.lower() for ext in hdu
                      if ext.name + '_ivar' in hdu and ext.name + '_mask']

        for category in categories:
            extension = hdu[category]
            if len(extension.data.shape) == 2:
                cat_dict[category] = None

            elif len(extension.data.shape) == 3:

                # Gets the channels and creates the names.
                channel_keys = [key for key in extension.header.keys()
                                if re.match('C[0-9]+', key)]
                names = [re.sub('\-+', '-', extension.header[key])
                         for key in channel_keys]

                cat_dict[category] = [name.lower() for name in names]

        return cat_dict

    else:

        mdb = marvin.marvindb

        # Emission lines
        emline_params = ['emline_' + row.name.lower()
                         for row in mdb.session.query(mdb.dapdb.EmLineParameter).all()]
        emline_channels = ['{0}-{1}'.format(row.name.lower(), row.rest_wavelength)
                           for row in mdb.session.query(mdb.dapdb.EmLineType).all()]
        for param in emline_params:
            cat_dict[param] = emline_channels

        # Stellar kinematics
        stell_kin_params = ['stellar_' + row.name.lower()
                            for row in mdb.session.query(mdb.dapdb.StellarKinParameter).all()]
        for param in stell_kin_params:
            cat_dict[param] = None

        # Spectral indices
        specindex_channels = [row.name.lower()
                              for row in mdb.session.query(mdb.dapdb.SpecIndexType).all()]
        cat_dict['specindex'] = specindex_channels

    return cat_dict
<|endoftext|>