<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2015 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
from contextlib import contextmanager
import stat
import re

import requests
from paramiko import SFTPAttributes

from jsonpath_rw import parse

from .abstract import AbstractConnection
import cumulus
from cumulus.common import check_status

newt_base_url = 'https://newt.nersc.gov/newt'
newt_stat_command = '/bin/stat -c "st_mode=%f,st_ino=%i,st_dev=%d,' \
    'st_nlink=%h,st_uid=%u,st_gid=%g,st_size=%s,st_atime=%X,st_mtime=%Y,' \
    'st_ctime=%Z" '
newt_mkdir_path = '/bin/mkdir'
newt_rm_path = '/bin/rm'

# TODO create map of command to fullpath for NEWT for example ls => /bin/ls ...
# then do the substiution ...

commands = {
    'ls': '/bin/ls',
    'rm': '/bin/rm'
}

type = {
    'd': stat.S_IFDIR,
    'l': stat.S_IFLNK
}

user = {
    'r': stat.S_IRUSR,
    'w': stat.S_IWUSR,
    'x': stat.S_IXUSR
}

group = {
    'r': stat.S_IRGRP,
    'w': stat.S_IWGRP,
    'x': stat.S_IXGRP
}

other = {
    'r': stat.S_IROTH,
    'w': stat.S_IWOTH,
    'x': stat.S_IXOTH
}

class NewtException(Exception):
    pass

class NewtClusterConnection(AbstractConnection):
    def __init__(self, girder_token, cluster):
        self._girder_token = girder_token
        self._cluster = cluster
        self._newt_session_id = None
        self._machine = parse('config.host').find(cluster)[0].value

    def __enter__(self):

        # Do we need to get the session id for this user
        if not self._newt_session_id:
            headers = {'Girder-Token':  self._girder_token}
            url = '%s/user/me' % cumulus.config.girder.baseUrl
            r = requests.get(url, headers=headers)
            check_status(r)

            session_id = parse('newt.sessionId').find(r.json())

            if not session_id:
                raise Exception('No NEWT session ID present')

            self._session = requests.Session()
            self._newt_session_id = session_id[0].value
            self._session.cookies.set('newt_sessionid', self._newt_session_id)

        return self

    def __exit__(self, type, value, traceback):
        pass

    def execute(self, command, ignore_exit_status=False, source_profile=True):
        url = '%s/command/%s' % (newt_base_url, self._machine)

        # NEWT requires all commands are issued using a full executable path
        for (name, full_path) in commands.iteritems():
            command = re.sub(r'^%s ' % name, '%s ' % full_path, command)

        data = {
            'executable': command,
            'loginenv': source_profile
        }
        r = self._session.post(url, data=data)
        check_status(r)

        json_response = r.json()
        if json_response['error']:
            raise NewtException(json_response['error'])

        return json_response['output']

    @contextmanager
    def get(self, remote_path):
        url = '%s/file/%s/%s' % (newt_base_url, self._machine, remote_path)
        params = {
            'view': 'read'
        }
        r = None

        try:
            r = self._session.get(url, params=params, stream=True)
            check_status(r)

            yield r.raw
        finally:
            if r:
                r.close()

    def isfile(self, remote_path):
        s = self.stat(remote_path)

        return not stat.S_ISDIR(s.st_mode)

    # TODO remote_path probably needs to be a full path
    def mkdir(self, remote_path, ignore_failure=False):
        command = newt_mkdir_path
        if ignore_failure:
            command += ' -p'
        command += ' %s' % remote_path

        return self.execute(command)

    def put(self, stream, remote_path):
        name = os.path.basename(remote_path)
        path = os.path.dirname(remote_path)

        files = {
            'file': (name, stream)
        }
        url = '%s/file/%s%s' % (newt_base_url, self._machine, path)
        r = self._session.post(url, files=files)
        check_status(r)

    def stat(self, remote_path):
        output = self.execute(newt_stat_command + remote_path)
        values = dict(s.split('=') for s in output.split(','))
        attributes = SFTPAttributes()
        for (key, value) in values.iteritems():
            try:
                value = int(value)
            except ValueError:
                value = int(value, 16)

            setattr(attributes, key, value)

        return attributes

    def remove(self, remote_path):
        command = newt_rm_path + ' %s' % remote_path

        return self.execute(command)

    def _perms_to_mode(self, perms):
        mode = 0
        index = 0

        def apply_perms(perms_to_modes, perms):
            mode = 0
            for p in perms:
                if p in perms_to_modes:
                    mode |= perms_to_modes[p]

            return mode

        # type
        mode |= apply_perms(type, perms[index:1])
        index += 1

        # user
        mode |= apply_perms(user, perms[index: index+3])
        index += 3

        # group
        mode |= apply_perms(group, perms[index: index+3])
        index += 3

        # other
        mode |= apply_perms(other, perms[index: index+3])
        index += 3

        return mode

    def list(self, remote_path):
        url = '%s/file/%s/%s' % (newt_base_url, self._machine, remote_path)

        r = self._session.get(url)
        check_status(r)

        paths = r.json()

        for path in paths:
            perms = path['perms']
            del path['perms']
            del path['hardlinks']

            path['mode'] = self._perms_to_mode(perms)
            yield path
<commit_msg>Remove TODO comment<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2015 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
from contextlib import contextmanager
import stat
import re

import requests
from paramiko import SFTPAttributes

from jsonpath_rw import parse

from .abstract import AbstractConnection
import cumulus
from cumulus.common import check_status

newt_base_url = 'https://newt.nersc.gov/newt'
newt_stat_command = '/bin/stat -c "st_mode=%f,st_ino=%i,st_dev=%d,' \
    'st_nlink=%h,st_uid=%u,st_gid=%g,st_size=%s,st_atime=%X,st_mtime=%Y,' \
    'st_ctime=%Z" '
newt_mkdir_path = '/bin/mkdir'
newt_rm_path = '/bin/rm'

commands = {
    'ls': '/bin/ls',
    'rm': '/bin/rm'
}

type = {
    'd': stat.S_IFDIR,
    'l': stat.S_IFLNK
}

user = {
    'r': stat.S_IRUSR,
    'w': stat.S_IWUSR,
    'x': stat.S_IXUSR
}

group = {
    'r': stat.S_IRGRP,
    'w': stat.S_IWGRP,
    'x': stat.S_IXGRP
}

other = {
    'r': stat.S_IROTH,
    'w': stat.S_IWOTH,
    'x': stat.S_IXOTH
}

class NewtException(Exception):
    pass

class NewtClusterConnection(AbstractConnection):
    def __init__(self, girder_token, cluster):
        self._girder_token = girder_token
        self._cluster = cluster
        self._newt_session_id = None
        self._machine = parse('config.host').find(cluster)[0].value

    def __enter__(self):

        # Do we need to get the session id for this user
        if not self._newt_session_id:
            headers = {'Girder-Token':  self._girder_token}
            url = '%s/user/me' % cumulus.config.girder.baseUrl
            r = requests.get(url, headers=headers)
            check_status(r)

            session_id = parse('newt.sessionId').find(r.json())

            if not session_id:
                raise Exception('No NEWT session ID present')

            self._session = requests.Session()
            self._newt_session_id = session_id[0].value
            self._session.cookies.set('newt_sessionid', self._newt_session_id)

        return self

    def __exit__(self, type, value, traceback):
        pass

    def execute(self, command, ignore_exit_status=False, source_profile=True):
        url = '%s/command/%s' % (newt_base_url, self._machine)

        # NEWT requires all commands are issued using a full executable path
        for (name, full_path) in commands.iteritems():
            command = re.sub(r'^%s ' % name, '%s ' % full_path, command)

        data = {
            'executable': command,
            'loginenv': source_profile
        }
        r = self._session.post(url, data=data)
        check_status(r)

        json_response = r.json()
        if json_response['error']:
            raise NewtException(json_response['error'])

        return json_response['output']

    @contextmanager
    def get(self, remote_path):
        url = '%s/file/%s/%s' % (newt_base_url, self._machine, remote_path)
        params = {
            'view': 'read'
        }
        r = None

        try:
            r = self._session.get(url, params=params, stream=True)
            check_status(r)

            yield r.raw
        finally:
            if r:
                r.close()

    def isfile(self, remote_path):
        s = self.stat(remote_path)

        return not stat.S_ISDIR(s.st_mode)

    # TODO remote_path probably needs to be a full path
    def mkdir(self, remote_path, ignore_failure=False):
        command = newt_mkdir_path
        if ignore_failure:
            command += ' -p'
        command += ' %s' % remote_path

        return self.execute(command)

    def put(self, stream, remote_path):
        name = os.path.basename(remote_path)
        path = os.path.dirname(remote_path)

        files = {
            'file': (name, stream)
        }
        url = '%s/file/%s%s' % (newt_base_url, self._machine, path)
        r = self._session.post(url, files=files)
        check_status(r)

    def stat(self, remote_path):
        output = self.execute(newt_stat_command + remote_path)
        values = dict(s.split('=') for s in output.split(','))
        attributes = SFTPAttributes()
        for (key, value) in values.iteritems():
            try:
                value = int(value)
            except ValueError:
                value = int(value, 16)

            setattr(attributes, key, value)

        return attributes

    def remove(self, remote_path):
        command = newt_rm_path + ' %s' % remote_path

        return self.execute(command)

    def _perms_to_mode(self, perms):
        mode = 0
        index = 0

        def apply_perms(perms_to_modes, perms):
            mode = 0
            for p in perms:
                if p in perms_to_modes:
                    mode |= perms_to_modes[p]

            return mode

        # type
        mode |= apply_perms(type, perms[index:1])
        index += 1

        # user
        mode |= apply_perms(user, perms[index: index+3])
        index += 3

        # group
        mode |= apply_perms(group, perms[index: index+3])
        index += 3

        # other
        mode |= apply_perms(other, perms[index: index+3])
        index += 3

        return mode

    def list(self, remote_path):
        url = '%s/file/%s/%s' % (newt_base_url, self._machine, remote_path)

        r = self._session.get(url)
        check_status(r)

        paths = r.json()

        for path in paths:
            perms = path['perms']
            del path['perms']
            del path['hardlinks']

            path['mode'] = self._perms_to_mode(perms)
            yield path
<|endoftext|>


<commit_before># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy

import cupy

import six

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = cupy.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = cupy.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = cupy.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, cupy.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return cupy.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(self, -1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]])

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__(self)

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6])

"""

# TODO(okuta): Implement indices

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = cupy.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

# TODO(okuta): Implement unravel_index

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<commit_msg>Do not use shortcut import<commit_after># flake8: NOQA
# "flake8: NOQA" to suppress warning "H104  File contains nothing but comments"

# class s_(object):

import numpy
import six

from cupy.creation import from_data

class AxisConcatenator(object):
    """Translates slice objects to concatenation along an axis.

    For detailed documentation on usage, see :func:`r_`.
    This implementation is partially borrowed from NumPy's one.

    """

    def _output_obj(self, obj, ndim, ndmin, trans1d):
        k2 = ndmin - ndim
        if trans1d < 0:
            trans1d += k2 + 1
        defaxes = list(six.moves.range(ndmin))
        k1 = trans1d
        axes = defaxes[:k1] + defaxes[k2:] + \
            defaxes[k1:k2]
        return obj.transpose(axes)

    def __init__(self, axis=0, matrix=False, ndmin=1, trans1d=-1):
        self.axis = axis
        self.trans1d = trans1d
        self.matrix = matrix
        self.ndmin = ndmin

    def __getitem__(self, key):
        trans1d = self.trans1d
        ndmin = self.ndmin
        objs = []
        scalars = []
        arraytypes = []
        scalartypes = []
        if isinstance(key, six.string_types):
            raise NotImplementedError
        if not isinstance(key, tuple):
            key = (key,)

        for i, k in enumerate(key):
            scalar = False
            if isinstance(k, slice):
                raise NotImplementedError
            elif isinstance(k, six.string_types):
                if i != 0:
                    raise ValueError(
                        'special directives must be the first entry.')
                raise NotImplementedError
            elif type(k) in numpy.ScalarType:
                newobj = from_data.array(k, ndmin=ndmin)
                scalars.append(i)
                scalar = True
                scalartypes.append(newobj.dtype)
            else:
                newobj = from_data.array(k, copy=False, ndmin=ndmin)
                if ndmin > 1:
                    ndim = from_data.array(k, copy=False).ndim
                    if trans1d != -1 and ndim < ndmin:
                        newobj = self._output_obj(newobj, ndim, ndmin, trans1d)

            objs.append(newobj)
            if not scalar and isinstance(newobj, cupy.ndarray):
                arraytypes.append(newobj.dtype)

        final_dtype = numpy.find_common_type(arraytypes, scalartypes)
        if final_dtype is not None:
            for k in scalars:
                objs[k] = objs[k].astype(final_dtype)

        return cupy.concatenate(tuple(objs), axis=self.axis)

    def __len__(self):
        return 0

class CClass(AxisConcatenator):

    def __init__(self):
        super(CClass, self).__init__(self, -1, ndmin=2, trans1d=0)

c_ = CClass()
"""Translates slice objects to concatenation along the second axis.

This is a CuPy object that corresponds to :func:`cupy.r_`, which is
useful because of its common occurrence. In particular, arrays will be
stacked along their last axis after being upgraded to at least 2-D with
1's post-pended to the shape (column vectors made out of 1-D arrays).

For detailed documentation, see :func:`r_`.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.c_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.c_[a, 0, 0, b]
array([[1, 2, 3, 0, 0, 4, 5, 6]])

"""

class RClass(AxisConcatenator):

    def __init__(self):
        super(RClass, self).__init__(self)

r_ = RClass()
"""Translates slice objects to concatenation along the first axis.

This is a simple way to build up arrays quickly.
If the index expression contains comma separated arrays, then stack
them along their first axis.

This object can build up from normal CuPy arrays.
Therefore, the other objects (e.g. writing strings like '2,3,4',
or using imaginary numbers like [1,2,3j],
or using string integers like '-1') are not implemented yet
compared with NumPy.

This implementation is partially borrowed from NumPy's one.

Args:
    Not a function, so takes no parameters

Returns:
    cupy.ndarray: Joined array.

.. seealso:: :func:`numpy.r_`

Examples
--------
>>> a = cupy.array([1, 2, 3], dtype=np.int32)
>>> b = cupy.array([4, 5, 6], dtype=np.int32)
>>> cupy.r_[a, 0, 0, b]
array([1, 2, 3, 0, 0, 4, 5, 6])

"""

# TODO(okuta): Implement indices

def ix_(*args):
    """Construct an open mesh from multiple sequences.

    This function takes N 1-D sequences and returns N outputs with N
    dimensions each, such that the shape is 1 in all but one dimension
    and the dimension with the non-unit shape value cycles through all
    N dimensions.

    Using `ix_` one can quickly construct index arrays that will index
    the cross product. ``a[cupy.ix_([1,3],[2,5])]`` returns the array
    ``[[a[1,2] a[1,5]], [a[3,2] a[3,5]]]``.

    Args:
        *args: 1-D sequences

    Returns:
        tuple of ndarrays:
        N arrays with N dimensions each, with N the number of input sequences.
        Together these arrays form an open mesh.

    Examples
    --------
    >>> a = cupy.arange(10).reshape(2, 5)
    >>> a
    array([[0, 1, 2, 3, 4],
           [5, 6, 7, 8, 9]])
    >>> ixgrid = cupy.ix_([0,1], [2,4])
    >>> ixgrid
    (array([[0],
           [1]]), array([[2, 4]]))

     .. seealso:: :func:`numpy.ix_`

    """
    out = []
    nd = len(args)
    for k, new in enumerate(args):
        new = cupy.asarray(new)
        if new.ndim != 1:
            raise ValueError("Cross index must be 1 dimensional")
        if new.size == 0:
            # Explicitly type empty arrays to avoid float default
            new = new.astype(numpy.intp)
        if cupy.issubdtype(new.dtype, cupy.bool_):
            new, = new.nonzero()
        new = new.reshape((1,) * k + (new.size,) + (1,) * (nd - k - 1))
        out.append(new)
    return tuple(out)

# TODO(okuta): Implement ravel_multi_index

# TODO(okuta): Implement unravel_index

# TODO(okuta): Implement diag_indices

# TODO(okuta): Implement diag_indices_from

# TODO(okuta): Implement mask_indices

# TODO(okuta): Implement tril_indices

# TODO(okuta): Implement tril_indices_from

# TODO(okuta): Implement triu_indices

# TODO(okuta): Implement triu_indices_from
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

Author
------
Bo Zhang

Email
-----
bozhang@nao.cas.cn

Created on
----------
- Tue Mar  8 15:26:00 2016    read_spectrum

Modifications
-------------
- Fri Jul 15 16:08:00 2016    migrate read_spectrum from lamost.py

Aims
----
- read various kinds of spectra

"""

import os

import numpy as np
from astropy.io import fits
from astropy.table import Table, Column

from .lamost import lamost_filepath
from .spec import Spec

def reconstruct_wcs_coord_from_fits_header(hdr, dim=1):
    """ reconstruct wcs coordinates (e.g., wavelength array) """
    # assert dim is not larger than limit
    assert dim <= hdr['NAXIS']

    # get keywords
    crval = hdr['CRVAL%d' % dim]
    cdelt = hdr['CDELT%d' % dim]
    try:
        crpix = hdr['CRPIX%d' % dim]
    except KeyError:
        crpix = 1

    # length of the current dimension
    naxis_ = hdr['NAXIS%d' % dim]

    # reconstruct wcs coordinates
    coord = np.arange(1 - crpix, naxis_ + 1 - crpix) * cdelt + crval
    return coord

def read_spectrum_phoenix_r10000(fp):
    """ read spectrum from PHOENIX R10000 """
    hl = fits.open(fp)
    wave = np.e ** reconstruct_wcs_coord_from_fits_header(hl[0].header)
    flux = hl[0].data
    return Table(data=[wave, flux], names=['wave', 'flux'])

def read_spectrum_elodie_r42000(fp):
    """ read spectrum from ELODIE library (R42000) """
    # assert the file exists
    assert os.path.exists(fp)

    # read fits
    hl = fits.open(fp)

    # reconstruct wave array
    wave = reconstruct_wcs_coord_from_fits_header(hl[0].header, dim=1)
    # flux
    flux = hl[0].data
    # flux err
    flux_err = hl[2].data
    # flux ivar
    flux_ivar = 1 / flux_err ** 2.

    # reconstruct spec
    sp = Spec(data=[wave, flux, flux_ivar, flux_err],
              names=['wave', 'flux', 'flux_ivar', 'flux_err'])
    return sp

def read_spectrum(filepath, filesource='auto'):
    """read SDSS/LAMOST spectrum

    Parameters
    ----------

    filepath: string
        input file path

    filesource: string
        {'sdss_dr12' / 'lamost_dr2' / 'lamost_dr3'}

    Returns
    -------

    specdata: astropy.table.Table
        spectra as a table

    """
    # auto-identify the spectrum origination
    if filesource == 'auto':
        telescope = fits.open(filepath)[0].header['TELESCOP']
        if telescope == 'SDSS 2.5-M':
            return read_spectrum(filepath, filesource='sdss_dr12')
        if telescope == 'LAMOST':
            return read_spectrum(filepath, filesource='lamost_dr3')

    # SDSS DR7 spectrum
    if filesource == 'sdss_dr7':
        hdulist = fits.open(filepath)

        # 1. flux, flux_err, mask
        data = hdulist[0].data  # 5 rows
        flux = data[0][:]
        flux_err = data[2][:]
        mask = data[3][:]

        # 2. wave
        # http://iraf.net/irafdocs/specwcs.php
        # wi = CRVALi + CDi_i * (li - CRPIXi)
        CRVAL1 = hdulist[0].header['CRVAL1']
        CD1_1 = hdulist[0].header['CD1_1']
        CRPIX1 = hdulist[0].header['CRPIX1']
        Npix = len(flux)
        wavelog = CRVAL1 + (np.arange(Npix) + 1 - CRPIX1) * CD1_1
        wave = np.power(10, wavelog)

        spec = Table([wave, flux, flux_err, mask],
                     names=['wave', 'flux', 'flux_err', 'mask'])
        return spec

    # SDSS DR10/DR12 spectrum
    if filesource == 'sdss_dr10' or filesource == 'sdss_dr12':
        data = fits.open(filepath)
        specdata = Table(data[1].data)
        wave = Column(name='wave', data=np.power(10., specdata['loglam']))
        flux_err = Column(name='flux_err', data=(specdata['ivar']) ** -0.5)
        specdata.add_columns([wave, flux_err])
        return specdata

    # LAMOST DR2/DR3 spectrum
    if filesource == 'lamost_dr3' or filesource == 'lamost_dr2' or filesource == 'lamost_dr1':
        data = fits.open(filepath)
        specdata = Table(data[0].data.T)
        flux = Column(name='flux', data=specdata['col0'])
        ivar = Column(name='ivar', data=specdata['col1'])
        flux_err = Column(name='flux_err', data=(specdata['col1']) ** -0.5)
        wave = Column(name='wave', data=specdata['col2'])
        and_mask = Column(name='and_mask', data=specdata['col3'])
        or_mask = Column(name='or_mask', data=specdata['col4'])
        # for flux_err, convert inf to nan
        flux_err[np.isinf(flux_err.data)] = np.nan
        return Table([wave, flux, flux_err, ivar, and_mask, or_mask])

    return None

def _test_read_spectrum():
    fp = '/home/cham/PycharmProjects/bopy/bopy/data/test_spectra/lamost_dr3/'\
         + lamost_filepath('GAC_061N46_V3', 55939, 7, 78)
    print(fp)
    sp = read_spectrum(fp)
    sp.pprint()

if __name__ == '__main__':
    print('')
    print('@Cham: testing ''read_spectrum'' ...')
    _test_read_spectrum()

"""

#######################################
filesource:   'lamost_dr2'
#######################################
documents of data structures (LAMOST and SDSS spectra)
http://dr2.lamost.org/doc/data-production-description#toc_3

 RowNumber 	Data                Type
 1           Flux                float
 2           Inverse Variance 	float
 3           WaveLength          float
 4           Andmask             float
 5           Ormask              float

#######################################
filesource:   'sdss_dr7'
#######################################

http://classic.sdss.org/dr7/dm/flatFiles/spSpec.html

#######################################
filesource:   'sdss_dr10' / 'sdss_dr12'
#######################################

http://data.sdss3.org/datamodel/files/BOSS_SPECTRO_REDUX/RUN2D/spectra/PLATE4/spec.html

 HDU 0  : Header info from spPlate
 HDU 1  : Coadded spectrum from spPlate --> use this
 HDU 2  : Summary metadata copied from spAll
 HDU 3  : Line fitting metadata from spZline
 HDU 4+ : [Optional] Individual spCFrame spectra [B, R for each exposure]

HDU 0: Header keywords only

Copied from spPlate with the following additions/modifications:

   PLUG_RA, PLUG_DEC, THING_ID, FIBERID: added from spAll
   NEXP and EXPID*: modified to just include the frames which contributed to this fiber
   Removed keywords which apply only to single exposures

HDU 1 (extname COADD): Coadded Spectrum from spPlate

Binary table with columns:
 Required    Columns
 Col     Name        Type        Comment
 1       flux        float32 	coadded calibrated flux [10-17 ergs/s/cm2/Å]
 2       loglam      float32 	log10(wavelength [Å])
 3       ivar        float32 	inverse variance of flux
 4       and_mask 	int32       AND mask
 5       or_mask 	int32       OR mask
 6       wdisp       float32 	wavelength dispersion in pixel=dloglam units
 7       sky         float32 	subtracted sky flux [10-17 ergs/s/cm2/Å]
 8       model       float32 	pipeline best model fit used for classification and redshift

"""<commit_msg>added MedSpec<commit_after># -*- coding: utf-8 -*-
"""

Author
------
Bo Zhang

Email
-----
bozhang@nao.cas.cn

Created on
----------
- Tue Mar  8 15:26:00 2016    read_spectrum

Modifications
-------------
- Fri Jul 15 16:08:00 2016    migrate read_spectrum from lamost.py

Aims
----
- read various kinds of spectra

"""

import os
from collections import OrderedDict

import numpy as np
from astropy.io import fits
from astropy.table import Table, Column

from .lamost import lamost_filepath
from .spec import Spec

def reconstruct_wcs_coord_from_fits_header(hdr, dim=1):
    """ reconstruct wcs coordinates (e.g., wavelength array) """
    # assert dim is not larger than limit
    assert dim <= hdr['NAXIS']

    # get keywords
    crval = hdr['CRVAL%d' % dim]
    cdelt = hdr['CDELT%d' % dim]
    try:
        crpix = hdr['CRPIX%d' % dim]
    except KeyError:
        crpix = 1

    # length of the current dimension
    naxis_ = hdr['NAXIS%d' % dim]

    # reconstruct wcs coordinates
    coord = np.arange(1 - crpix, naxis_ + 1 - crpix) * cdelt + crval
    return coord

def read_spectrum_phoenix_r10000(fp):
    """ read spectrum from PHOENIX R10000 """
    hl = fits.open(fp)
    wave = np.e ** reconstruct_wcs_coord_from_fits_header(hl[0].header)
    flux = hl[0].data
    return Table(data=[wave, flux], names=['wave', 'flux'])

def read_spectrum_elodie_r42000(fp):
    """ read spectrum from ELODIE library (R42000) """
    # assert the file exists
    assert os.path.exists(fp)

    # read fits
    hl = fits.open(fp)

    # reconstruct wave array
    wave = reconstruct_wcs_coord_from_fits_header(hl[0].header, dim=1)
    # flux
    flux = hl[0].data
    # flux err
    flux_err = hl[2].data
    # flux ivar
    flux_ivar = 1 / flux_err ** 2.

    # reconstruct spec
    sp = Spec(data=[wave, flux, flux_ivar, flux_err],
              names=['wave', 'flux', 'flux_ivar', 'flux_err'])
    return sp

def read_spectrum(filepath, filesource='auto'):
    """read SDSS/LAMOST spectrum

    Parameters
    ----------

    filepath: string
        input file path

    filesource: string
        {'sdss_dr12' / 'lamost_dr2' / 'lamost_dr3'}

    Returns
    -------

    specdata: astropy.table.Table
        spectra as a table

    """
    # auto-identify the spectrum origination
    if filesource == 'auto':
        telescope = fits.open(filepath)[0].header['TELESCOP']
        if telescope == 'SDSS 2.5-M':
            return read_spectrum(filepath, filesource='sdss_dr12')
        if telescope == 'LAMOST':
            return read_spectrum(filepath, filesource='lamost_dr3')

    # SDSS DR7 spectrum
    if filesource == 'sdss_dr7':
        hdulist = fits.open(filepath)

        # 1. flux, flux_err, mask
        data = hdulist[0].data  # 5 rows
        flux = data[0][:]
        flux_err = data[2][:]
        mask = data[3][:]

        # 2. wave
        # http://iraf.net/irafdocs/specwcs.php
        # wi = CRVALi + CDi_i * (li - CRPIXi)
        CRVAL1 = hdulist[0].header['CRVAL1']
        CD1_1 = hdulist[0].header['CD1_1']
        CRPIX1 = hdulist[0].header['CRPIX1']
        Npix = len(flux)
        wavelog = CRVAL1 + (np.arange(Npix) + 1 - CRPIX1) * CD1_1
        wave = np.power(10, wavelog)

        spec = Table([wave, flux, flux_err, mask],
                     names=['wave', 'flux', 'flux_err', 'mask'])
        return spec

    # SDSS DR10/DR12 spectrum
    if filesource == 'sdss_dr10' or filesource == 'sdss_dr12':
        data = fits.open(filepath)
        specdata = Table(data[1].data)
        wave = Column(name='wave', data=np.power(10., specdata['loglam']))
        flux_err = Column(name='flux_err', data=(specdata['ivar']) ** -0.5)
        specdata.add_columns([wave, flux_err])
        return specdata

    # LAMOST DR2/DR3 spectrum
    if filesource == 'lamost_dr3' or filesource == 'lamost_dr2' or filesource == 'lamost_dr1':
        data = fits.open(filepath)
        specdata = Table(data[0].data.T)
        flux = Column(name='flux', data=specdata['col0'])
        ivar = Column(name='ivar', data=specdata['col1'])
        flux_err = Column(name='flux_err', data=(specdata['col1']) ** -0.5)
        wave = Column(name='wave', data=specdata['col2'])
        and_mask = Column(name='and_mask', data=specdata['col3'])
        or_mask = Column(name='or_mask', data=specdata['col4'])
        # for flux_err, convert inf to nan
        flux_err[np.isinf(flux_err.data)] = np.nan
        return Table([wave, flux, flux_err, ivar, and_mask, or_mask])

    return None

def _test_read_spectrum():
    fp = '/home/cham/PycharmProjects/bopy/bopy/data/test_spectra/lamost_dr3/'\
         + lamost_filepath('GAC_061N46_V3', 55939, 7, 78)
    print(fp)
    sp = read_spectrum(fp)
    sp.pprint()

class MedSpec(OrderedDict):
    """ for Median Resolution Spectrum """
    meta = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def __repr__(self):
        s = "No.    Name     Dimensions\n"
        for i, (k,v) in enumerate(self.items()):
            s += "{}   {}   {}Rx{}C\n".format(i, k, len(v), len(v.colnames))
        return s

    @staticmethod
    def read(fp):
        return read_lamostms(fp)

def read_lamostms(fp):
    # read fits
    hl = fits.open(fp)

    # initialize MS
    ms = MedSpec()

    # set meta
    ms.meta = OrderedDict(hl[0].header)

    # set spec
    for i, data in enumerate(hl[1:]):
        ms[data.name] = Table(data=data.data, meta=OrderedDict(data.header))

    return ms

if __name__ == '__main__':
    print('')
    print('@Cham: testing ''read_spectrum'' ...')
    _test_read_spectrum()
    print(MedSpec.read(fp="file:///home/cham/PycharmProjects/bopy/bopy/data/test_spectra/lamost_dr7/med-58417-NT062347N023956S01_sp01-008.fits"))

"""

#######################################
filesource:   'lamost_dr2'
#######################################
documents of data structures (LAMOST and SDSS spectra)
http://dr2.lamost.org/doc/data-production-description#toc_3

 RowNumber 	Data                Type
 1           Flux                float
 2           Inverse Variance 	float
 3           WaveLength          float
 4           Andmask             float
 5           Ormask              float

#######################################
filesource:   'sdss_dr7'
#######################################

http://classic.sdss.org/dr7/dm/flatFiles/spSpec.html

#######################################
filesource:   'sdss_dr10' / 'sdss_dr12'
#######################################

http://data.sdss3.org/datamodel/files/BOSS_SPECTRO_REDUX/RUN2D/spectra/PLATE4/spec.html

 HDU 0  : Header info from spPlate
 HDU 1  : Coadded spectrum from spPlate --> use this
 HDU 2  : Summary metadata copied from spAll
 HDU 3  : Line fitting metadata from spZline
 HDU 4+ : [Optional] Individual spCFrame spectra [B, R for each exposure]

HDU 0: Header keywords only

Copied from spPlate with the following additions/modifications:

   PLUG_RA, PLUG_DEC, THING_ID, FIBERID: added from spAll
   NEXP and EXPID*: modified to just include the frames which contributed to this fiber
   Removed keywords which apply only to single exposures

HDU 1 (extname COADD): Coadded Spectrum from spPlate

Binary table with columns:
 Required    Columns
 Col     Name        Type        Comment
 1       flux        float32 	coadded calibrated flux [10-17 ergs/s/cm2/Å]
 2       loglam      float32 	log10(wavelength [Å])
 3       ivar        float32 	inverse variance of flux
 4       and_mask 	int32       AND mask
 5       or_mask 	int32       OR mask
 6       wdisp       float32 	wavelength dispersion in pixel=dloglam units
 7       sky         float32 	subtracted sky flux [10-17 ergs/s/cm2/Å]
 8       model       float32 	pipeline best model fit used for classification and redshift

"""<|endoftext|>


<commit_before># Django settings for bostongreenmap project.

from os.path import join, abspath, dirname
import os
import sys
# Defines the parent directory that contains the settings file as
# the PROJECT_ROOT. It is used for relative settings elsewhere.
here = lambda *x: join(abspath(dirname(__file__)), *x)
PROJECT_ROOT = here('..')
root = lambda *x: join(abspath(PROJECT_ROOT), *x)

DEBUG = True
TEMPLATE_DEBUG = DEBUG

LOGIN_REDIRECT_URL = '/login_redirect'

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = root('media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = root('collected_static')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    root('static'),
    root('client')
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '2nq6(xmm9pp^%$k9=$qq28$kj$w!g^7na_0@e%(658#+7qj1ho'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'bostongreenmap.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'bostongreenmap.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    root('templates'),
    root('client')
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    # 'django.contrib.staticfiles',
    'grappelli',
    'django.contrib.admin',
    'django.contrib.markup',
    'django.contrib.humanize',
    'django.contrib.gis',
    # 'registration',
    'south',
    'parks',
    'sorl.thumbnail',
)

AUTH_PROFILE_MODULE = 'accounts.UserProfile'

# Django registration requires this.  It is the number of
# days after a new user has registered by which he must
# activate his account.  There is no default.
ACCOUNT_ACTIVATION_DAYS = 5
# DEFAULT_FROM_EMAIL must also be set, and such other EMAIL
# configuration as to enable successful operation of
#   django.contrib.auth.models.User().email_user
# Using the default smtp email backend, this requires
# EMAIL_HOST, the default is 'localhost', which may or may
# not cut it.  The default EMAIL_PORT is probably correct.
# EMAIL_HOST_USER adn EMAIL_HOST_PASSWORD each default to
# an empty string, which may be ok with the default
# EMAIL_HOST.  EMAIL_USE_TLS defaults to false.

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
            'datefmt' : "%d/%b/%Y %H:%M:%S"
        },
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': root('logfile.log'),
            'maxBytes': 50000,
            'backupCount': 2,
            'formatter': 'standard',
        },
        'console':{
            'level':'INFO',
            'class':'logging.StreamHandler',
            'stream': sys.stdout
        },
    },
    'loggers': {
        'django': {
            'handlers':['console'],
            'propagate': True,
            'level':'WARN',
        },
        'django.db.backends': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'parks': {
            'handlers': ['console', 'logfile'],
            'level': 'DEBUG',
        },
    }
}

ADMIN_THUMBS_SIZE = '160x120'

# 3rd party services
GA_TRACKING_ID = ''
USERVOICE_FORUM_ID = ''

# heroku/dokku settings
if os.environ.get('IS_OKU'):
    load_settings('oku_settings.py')

# import local settings
try:
    from local_settings import *
except ImportError:
    pass

MBTA_DISTANCE = 0.5 # Distance to search for MBTA stops on Park Page and Trips.
<commit_msg>Settings didn't have load_settings defined.<commit_after># Django settings for bostongreenmap project.

from os.path import join, abspath, dirname
import os
import sys
# Defines the parent directory that contains the settings file as
# the PROJECT_ROOT. It is used for relative settings elsewhere.
here = lambda *x: join(abspath(dirname(__file__)), *x)
PROJECT_ROOT = here('..')
root = lambda *x: join(abspath(PROJECT_ROOT), *x)

DEBUG = True
TEMPLATE_DEBUG = DEBUG

LOGIN_REDIRECT_URL = '/login_redirect'

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = root('media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = root('collected_static')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    root('static'),
    root('client')
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '2nq6(xmm9pp^%$k9=$qq28$kj$w!g^7na_0@e%(658#+7qj1ho'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'bostongreenmap.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'bostongreenmap.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    root('templates'),
    root('client')
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    # 'django.contrib.staticfiles',
    'grappelli',
    'django.contrib.admin',
    'django.contrib.markup',
    'django.contrib.humanize',
    'django.contrib.gis',
    # 'registration',
    'south',
    'parks',
    'sorl.thumbnail',
)

AUTH_PROFILE_MODULE = 'accounts.UserProfile'

# Django registration requires this.  It is the number of
# days after a new user has registered by which he must
# activate his account.  There is no default.
ACCOUNT_ACTIVATION_DAYS = 5
# DEFAULT_FROM_EMAIL must also be set, and such other EMAIL
# configuration as to enable successful operation of
#   django.contrib.auth.models.User().email_user
# Using the default smtp email backend, this requires
# EMAIL_HOST, the default is 'localhost', which may or may
# not cut it.  The default EMAIL_PORT is probably correct.
# EMAIL_HOST_USER adn EMAIL_HOST_PASSWORD each default to
# an empty string, which may be ok with the default
# EMAIL_HOST.  EMAIL_USE_TLS defaults to false.

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
            'datefmt' : "%d/%b/%Y %H:%M:%S"
        },
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': root('logfile.log'),
            'maxBytes': 50000,
            'backupCount': 2,
            'formatter': 'standard',
        },
        'console':{
            'level':'INFO',
            'class':'logging.StreamHandler',
            'stream': sys.stdout
        },
    },
    'loggers': {
        'django': {
            'handlers':['console'],
            'propagate': True,
            'level':'WARN',
        },
        'django.db.backends': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'parks': {
            'handlers': ['console', 'logfile'],
            'level': 'DEBUG',
        },
    }
}

ADMIN_THUMBS_SIZE = '160x120'

# 3rd party services
GA_TRACKING_ID = ''
USERVOICE_FORUM_ID = ''

def load_settings(settings_filename):
    settings_filename = os.path.join(SETTINGS_DIR, settings_filename)
    with open(settings_filename, 'r') as settings_file:
        exec(settings_file.read(), globals())

# heroku/dokku settings
if os.environ.get('IS_OKU'):
    load_settings('oku_settings.py')

# import local settings
try:
    from local_settings import *
except ImportError:
    pass

MBTA_DISTANCE = 0.5 # Distance to search for MBTA stops on Park Page and Trips.
<|endoftext|>


<commit_before># Module adapting param classes for use with boxflow
#
#
from __future__ import absolute_import
import param

from .interface import Interface, BoxType

class ParamBox(param.Parameterized):

    def propagate(self):
        return getattr(self, self.__class__.__name__.lower())

class Number(ParamBox):
    number = param.Number(default=0)

class Integer(ParamBox):
    integer = param.Integer(default=0)

class String(ParamBox):
    string = param.String(default='')

def load_param():
    boxtypes = [BoxType(p, hidden=[p.name.lower()]) for p in [ Number, Integer, String]]
    Interface.add('param', boxtypes)
<commit_msg>Added param.Time<commit_after># Module adapting param classes for use with boxflow
#
#
from __future__ import absolute_import
import param
import fractions

from .interface import Interface, BoxType

param.Dynamic.time_fn(val=0.0, time_type=fractions.Fraction)
param.Dynamic.time_dependent = True

class ParamBox(param.Parameterized):

    def propagate(self):
        return getattr(self, self.__class__.__name__.lower())

class Number(ParamBox):
    number = param.Number(default=0)

class Integer(ParamBox):
    integer = param.Integer(default=0)

class String(ParamBox):
    string = param.String(default='')

class Time(param.Parameterized):

    time = param.Number(default=0)

    def propagate(self):
        return float(param.Dynamic.time_fn(fractions.Fraction(self.time)))

def load_param():
    boxtypes = [BoxType(p, hidden=[p.name.lower()]) for p in [ Number, Integer, String]]
    Interface.add('param', boxtypes + [Time])
<|endoftext|>


<commit_before># Sigma's internal version control.
sigma_version = 'Beta 0.41b'
# 0 - Bot Application; 1 - User Account; Must be either 0 or 1.
StartupType = '0'
# Bot Application Token; Must be filled if the startup type is 0.
Token = ''
# GitHub WebHook Server Settings.
GitHubWebserverPort = ''
GitHubWebserverAddr = ''
# Discord User Account login details; Must be filled if the startup type is 1.
dsc_email = ''
dsc_password = ''
# API Keys and Login Details for Various Services.
OpenWeatherMapKey = ''
MashapeKey = ''
RiotAPIKey = ''
GoogleAPIKey = ''
LastFMAPIKey = ''
ITADKey = ''
SteamAPI = ''
SonarrKey = ''
BlizzardKey = ''
RLAPIKey = ''
ImgurClientID = ''
ImgurClientSecret = ''
WarGamingAppID = ''
mal_un = ''
mal_pw = ''
reddit_un = ''
reddit_pw = ''
# Bot Prefix for Commands.
Prefix = '>>'
# Aurora Project Donators.
donators = []
# Permitted IDs and Roles.
permitted_id = ['1234567891234569']
permitted_roles = []
<commit_msg>add ClientID to config_example.py<commit_after># Sigma's internal version control.
sigma_version = 'Beta 0.41b'
# 0 - Bot Application; 1 - User Account; Must be either 0 or 1.
StartupType = '0'
# Bot Application Token; Must be filled if the startup type is 0.
Token = ''
ClientID = ''
# GitHub WebHook Server Settings.
GitHubWebserverPort = 37812
GitHubWebserverAddr = ''
# Discord User Account login details; Must be filled if the startup type is 1.
dsc_email = ''
dsc_password = ''
# API Keys and Login Details for Various Services.
OpenWeatherMapKey = ''
MashapeKey = ''
RiotAPIKey = ''
GoogleAPIKey = ''
LastFMAPIKey = ''
ITADKey = ''
SteamAPI = ''
SonarrKey = ''
BlizzardKey = ''
RLAPIKey = ''
ImgurClientID = ''
ImgurClientSecret = ''
WarGamingAppID = ''
mal_un = ''
mal_pw = ''
reddit_un = ''
reddit_pw = ''
# Bot Prefix for Commands.
Prefix = '>>'
# Aurora Project Donators.
donators = []
# Permitted IDs and Roles.
permitted_id = ['1234567891234569']
permitted_roles = []
<|endoftext|>


<commit_before># Author Michele Mattioni
# Wed Mar 18 17:51:51 GMT 2009

import logging
import numpy as np
import math

import sys
import os

from neuron import h

import matplotlib
from ecellControl.ecellManager import EcellManager
backend = 'Agg'
matplotlib.use(backend)
import matplotlib.pyplot as plt

from neuronControl import nrnSim, synapse
from neuronControl.stimul import Stimul
import neuronControl

from neuronvisio.manager import Manager
from sumatra.external.NeuroTools import parameters

import tables
from extref import ExtRef

def build_vecs_to_plot(var, secs, anyRefs):
    """Create the dictionary of section->vectors to plot"""
    vecs_to_plot = {}
    for secName in secs:
        for ref in anyRefs:
            if secName == ref.sec_name:
                if ref.vecs.has_key(var):
                    vecs_to_plot[secName] = ref.vecs[var]
    return vecs_to_plot

def kir_gkbar(gkbar):
    for sec in h.allsec():
        for seg in sec:
            for mech in seg:
                if mech.name() == 'kir':
                    mech.gkbar = gkbar

def iClamptest(delay=10, duration=250, amplititude=0.248):
    """# Quick Iclamp test to check the current dynamics"""
    iclamp = h.IClamp(h.MSP_Cell[0].soma(0.5))
    iclamp.delay = delay
    iclamp.dur = duration
    iclamp.amp = amplititude

def advance_ecell(spine, delta_t):
    """
    Advance ecell simulator in `spine` of `delta_t`.

    Paramters:
    ----------
    tmp_tstop: Temporary tstop. It has to be expressed in seconds
    """
    current_time = spine.ecellMan.ses.getCurrentTime()
    len_current_time = len (spine.ecellMan.loggers['ca'].getData()[:,0])
    print ("Ecell current time: %s in %s. Advancing of: %s seconds.\
    Current time len: %s") %(current_time, spine.id, delta_t, len_current_time)
    spine.ecellMan.ses.run(delta_t)

def synch_simulators(tmp_tstop, stim_spines_id,
                     dtNeuron,
                     delta_calcium_sampling,
                     weight_baseline):
    """
    Calculate the synapse weight, using the calcium in the spine_heads
    as input.

    Synch the two simulators using the following steps:
    1. Calculate the calcium concentration in the spines head in
    NEURON and set this value in ecell.
    2. Advance ecell for the specified_delta
    3. Update the electric weight of the synapses in NEURON
    """
    print ("Synchronizing sims till [ms] %s") %tmp_tstop

    t_synch_start = h.t
    while h.t < tmp_tstop:
        h.fadvance() # run Neuron for step
        #for every ms in NEURON we update the ecellMan
        if np.round(h.t, decimals = 4) % delta_calcium_sampling == 0:
            for spine_id in stim_spines_id :
                spine = nrnSim.spines[spine_id]
                sync_calcium(spine, dtNeuron, delta_calcium_sampling)
                advance_ecell(spine, (h.t - t_synch_start) / 1e3)
                # Stopping flux from the input.
                spine.ecellMan.ca_in['k'] = 0
                # Re-enabling pump and leak.
                spine.ecellMan.ca_leak['vmax'] = param['ca_leak_vmax']
                spine.ecellMan.ca_pump['vmax'] = param['ca_pump_vmax']
                update_synape_weight(spine, weight_baseline)
            t_synch_start = h.t # Resetting the t_start to the new NEURON time.

def sync_calcium(spine, dtNeuron, delta_calcium_sampling):
    """"
    Calculate the flux of the calcium in the spine_head and synch
    it with ecell.
    """
    if hasattr(spine, 'ecellMan'):

        k_ca_flux = get_calcium_flux(dtNeuron,
                                     delta_calcium_sampling,
                                     spine)
        # Unit conversion in update_calcium
        spine.update_calcium(k_ca_flux)

def get_calcium_flux(dtNeuron, delta_calcium_sampling, spine):
    """
    Retrieving the calcium in the interval. end is always -1 because is
    the last timepoint available, start is when the interval has begun
    """
    start_index = -int(delta_calcium_sampling / dtNeuron)
    # Getting the calcium value
    vec_spine_head_cai = manager.get_vector(spine.head, 'cai')
    vec_spine_head_cali = manager.get_vector(spine.head, 'cali')

    head_cai = vec_spine_head_cai.x[start_index]
    head_cali = vec_spine_head_cali.x[start_index]
    electrical_ca_start = head_cai + head_cali

    head_cai = vec_spine_head_cai.x[-1]
    head_cali = vec_spine_head_cali.x[-1]
    electrical_ca_end = head_cai + head_cali
    electrical_diff = electrical_ca_end - electrical_ca_start
    print "Len vecs: %s start_idx: %s" %(len(vec_spine_head_cali), start_index)
    print "Electrical calcium start: %s end: %s difference: %s" %(electrical_ca_start,
                                                                  electrical_ca_end,
                                                                  electrical_diff)
    # Calculating the flux
    k_calcium_flux = electrical_diff / delta_calcium_sampling

    return k_calcium_flux

def update_synape_weight(spine, baseline):
    """
    Update the electrical weight's synapses.

    spine : the spine where the weight should be updated
    baseline : the equilibrium concentration of AMPAR-P which we use as reference
    and  put equal to one.
    """

    # Updating the AMPA synapses
    for synapse in spine.synapses:
        if synapse.chan_type == 'ampa':
        # Retrieve the value of the weight.
            weight = spine.ecellMan.ampar_P['Value']/baseline
            synapse.netCon.weight[0] = weight
            # The weight of the ampa is a double list
            # Check the specs in synapse weight for more info.
            synapse.weight[0].append(h.t)
            synapse.weight[1].append(weight)
            print "Updating synapse weight in %s, time [ms]: %s, weight: %s" %(spine.id,
                                                                               h.t,
                                                                               weight)

def create_excitatory_inputs(stim_spines_id, neuron_time_interval, param):
    """
    Create the excitatory inputs according to the parametes file.

    Create the NEURON inputs on each synapses according to the parameters.
    Set and initialize the ecell biochemical simulator in the stimulated
    spine."""

    excitatory_stimuli = []

    for spine_id in stim_spines_id:
        if spine_id in param.keys():
            spine = nrnSim.spines[spine_id]
            for stim_id in param[spine.id]:
                stim_dictionary = param[stim_id]
                stim = Stimul((stim_dictionary['t_stim']* 1e3),
                              stim_dictionary['numbers'],
                              stim_dictionary['delay'],
                              stim_dictionary['type'])
                if stim.chan_type == 'ampa':
                    for syn in spine.synapses:
                        if syn.chan_type == 'ampa':
                            syn.stims.append(stim)
                elif stim.chan_type == 'nmda':# more than one stim
                    for syn in spine.synapses:
                        if syn.chan_type == 'nmda':
                            syn.stims.append(stim)

                stims_time = stim.get_stims_time()
                excitatory_stimuli.extend(stims_time)

            spine.deploy_stims(neuron_time_interval)
            spine.setupBioSim() # Initializing ecell

    excitatory_stimuli = list(set(excitatory_stimuli))
    excitatory_stimuli.sort()
    return excitatory_stimuli

def advance_quickly(tmp_tstop, stim_spines_id, weight_baseline):
    """
    Advance the two simulators quickly in an independent way. Synapse weight
    is synchronized at the end
    """
    #Update the weight
    for spine_id in stim_spines_id:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

    delta_ecell = tmp_tstop - h.t
    delta_ecell_seconds = delta_ecell / 1e3
    print ("\nAdvance quickly routine.")
    print ("Current Neuron time: %s, aimed tstop[ms]: %s") %(h.t, tmp_tstop)
    print ("Delta applied on Ecell simulator [s]: %s\n") % delta_ecell_seconds
    nrnSim.run(tmp_tstop)
    for spine_id in stim_spines_id:
        spine = nrnSim.spines[spine_id]
        advance_ecell(spine, delta_ecell_seconds)
        update_synape_weight(spine, weight_baseline)

def run_simulation(tStop_final, t_buffer, dtNeuron,
                   delta_calcium_sampling, weight_baseline, param):
    """
    Run the simulation. If input synchronizes the two simulators,
    otherwise run each on its own and advance quickly

    """

    # Getting the calcium before the stims
    for spine_id in param['stimulated_spines']:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

    while h.t < tStop_final:

        if excitatory_stims:
            t_stim = excitatory_stims.pop(0)
            print "Current Neuron time: %s. Current t_stim: %s, remaining input: %s" %(h.t,
                                                                                t_stim,
                                                                                len(excitatory_stims))
            if h.t < t_stim:
                advance_quickly(t_stim, stim_spines_id, weight_baseline)
                tmp_tstop = t_stim + t_buffer
                synch_simulators(tmp_tstop, stim_spines_id,
                                 dtNeuron,
                                 delta_calcium_sampling,
                                 weight_baseline)

        else:
            print "No excitatory input remaining. Quickly to the end"
            advance_quickly(tStop_final, stim_spines_id, weight_baseline)
            h.fadvance() # This is to force the latest step and avoid the infinite loop.

    # Recording last
    for spine_id in param['stimulated_spines']:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

def main(argv):

    if len(argv) != 2:
        print("No parameter file supplied. Abort.")
        usage = 'python spineIntegration.py parameters_file.param'
        print usage
        sys.exit()

    parameter_file = argv[1]
    param = parameters.ParameterSet(parameter_file)

    # Processing the options
    h.dt = param['dtNeuron']
    delta_calcium_sampling = param['delta_calcium_sampling']
    t_equilibrium_neuron = param['tEquilibrium_neuron']
    t_equilibrium_ecell = param['tEquilibrium_ecell']

    tStop_final = param['tStop']

    print("Starting Spine integration")

    hoc_path = "hoc"
    mod_path="mod"

    nrnSim = neuronControl.NeuronSim(mod_path=mod_path, hoc_path=hoc_path,
                              spines_dist=param['spines_dist'],
                              biochemical_filename=param['biochemical_filename'])

    kir_gkbar(param['kir_gkbar'])

    #--------------------------------------------------------------------------
    # Recording and stimul

    # Create Manager
    manager = Manager()

    # Set the stimuls to the synapses

    stim_spines_id = param['stimulated_spines']
    excitatory_stims = create_excitatory_inputs(stim_spines_id,
                                                param['time_resolution_neuron'],
                                                param)
    print "This are the time of the stims: %s" %excitatory_stims

    # Recording the sections

    variables_to_rec = param['var_to_plot']

    for var in variables_to_rec:
        for sec in h.allsec():
            manager.add_all_vecRef(var, param['time_resolution_neuron']) # Saving all the vecRef for testing
#            if sec.name() in param['section_to_plot']:
#                manager.add_vecRef(var, sec, param['time_resolution_neuron'])

    # Recording the synapses
    for spine_id in stim_spines_id:
        spine = nrnSim.spines[spine_id]
        for syn in spine.synapses:
            manager.add_synVecRef(syn)

    ##------------------------------------------------------------------------------
    ## Experiment
    nrnSim.init() # Initializing neuron

    # equilibrium
    print ("#--#")
    print ("Equilibrium run for the two simulators")
    nrnSim.run(t_equilibrium_neuron)
    for spine_id in stim_spines_id:
        advance_ecell(nrnSim.spines[spine_id], t_equilibrium_ecell)
    print ("Equilibrium run finished. Starting normal simulation.")
    print ("#--#")
    t_buffer = param['t_buffer']
    run_simulation(tStop_final, t_buffer,
                   h.dt, delta_calcium_sampling, param['weight_baseline'],
                   param)

    #------------------------------------
    # Save the Results

    # Add timeseries
    extRef = ExtRef()
    extRef.add_timeseries(manager, param['stimulated_spines'], nrnSim)

    # Saving the weight
    extRef.add_weights(manager, param['stimulated_spines'], nrnSim)

    print "Simulation Ended. Saving results"
    saving_dir = manager.create_new_dir(root='Data')
    hdf_name = 'storage.h5'
    filename = os.path.join(saving_dir, hdf_name)
    print "Results will be saved in %s" %filename
    # Saving everything
    manager.save_to_hdf(filename)

    #Let's save same plot

    for i, var in enumerate(param['var_to_plot']):
        secs = param['section_to_plot']
        vecs_to_plot = build_vecs_to_plot(var, secs, manager.refs['VecRef'])
        manager.plotVecs(vecs_to_plot, figure_num=i)

        if var == 'v':
            plt.ylabel("Voltage [mV]")
            plt.xlabel("Time [ms]")
            plt.ylim(-90) # Setting the minimum limits

        elif var == 'cai' or var == 'cali':
            plt.xlabel("Time [ms]")
            plt.ylabel("Concentration [mM]")

        elif var == 'ica':
            plt.xlabel("Time [ms]")
            plt.ylabel("Current [nA]")

        fig_file = 'plot_' + var
        plt.savefig(os.path.join(saving_dir, fig_file))

    from ecellControl import ecellManager as eM
    x_start = param['tEquilibrium_ecell']
    x_stop = x_start + param['tStop']/1e3
    for stim_spine in param['stimulated_spines']:
        spine = nrnSim.spines[stim_spine]
        eM.plotTimeCourses(spine.ecellMan.timeCourses, save=True,
                           dir=saving_dir, name=spine.id,
                           x_lims= [x_start, x_stop])
        eM.plotWeight(spine.ecellMan.timeCourses, dir=saving_dir)

if __name__ == "__main__":
    main(sys.argv)

<commit_msg>Fixing the name of the neuron controller to a better one for sanity check.<commit_after># Author Michele Mattioni
# Wed Mar 18 17:51:51 GMT 2009

import logging
import numpy as np
import math

import sys
import os

from neuron import h

import matplotlib
from ecellControl.ecellManager import EcellManager
backend = 'Agg'
matplotlib.use(backend)
import matplotlib.pyplot as plt

from neuronControl import nrnSim, synapse
from neuronControl.stimul import Stimul
import neuronControl

from neuronvisio.manager import Manager
from sumatra.external.NeuroTools import parameters

import tables
from extref import ExtRef

def build_vecs_to_plot(var, secs, anyRefs):
    """Create the dictionary of section->vectors to plot"""
    vecs_to_plot = {}
    for secName in secs:
        for ref in anyRefs:
            if secName == ref.sec_name:
                if ref.vecs.has_key(var):
                    vecs_to_plot[secName] = ref.vecs[var]
    return vecs_to_plot

def kir_gkbar(gkbar):
    for sec in h.allsec():
        for seg in sec:
            for mech in seg:
                if mech.name() == 'kir':
                    mech.gkbar = gkbar

def iClamptest(delay=10, duration=250, amplititude=0.248):
    """# Quick Iclamp test to check the current dynamics"""
    iclamp = h.IClamp(h.MSP_Cell[0].soma(0.5))
    iclamp.delay = delay
    iclamp.dur = duration
    iclamp.amp = amplititude

def advance_ecell(spine, delta_t):
    """
    Advance ecell simulator in `spine` of `delta_t`.

    Paramters:
    ----------
    tmp_tstop: Temporary tstop. It has to be expressed in seconds
    """
    current_time = spine.ecellMan.ses.getCurrentTime()
    len_current_time = len (spine.ecellMan.loggers['ca'].getData()[:,0])
    print ("Ecell current time: %s in %s. Advancing of: %s seconds.\
    Current time len: %s") %(current_time, spine.id, delta_t, len_current_time)
    spine.ecellMan.ses.run(delta_t)

def synch_simulators(tmp_tstop, stim_spines_id,
                     dtNeuron,
                     delta_calcium_sampling,
                     weight_baseline):
    """
    Calculate the synapse weight, using the calcium in the spine_heads
    as input.

    Synch the two simulators using the following steps:
    1. Calculate the calcium concentration in the spines head in
    NEURON and set this value in ecell.
    2. Advance ecell for the specified_delta
    3. Update the electric weight of the synapses in NEURON
    """
    print ("Synchronizing sims till [ms] %s") %tmp_tstop

    t_synch_start = h.t
    while h.t < tmp_tstop:
        h.fadvance() # run Neuron for step
        #for every ms in NEURON we update the ecellMan
        if np.round(h.t, decimals = 4) % delta_calcium_sampling == 0:
            for spine_id in stim_spines_id :
                spine = nrnSim.spines[spine_id]
                sync_calcium(spine, dtNeuron, delta_calcium_sampling)
                advance_ecell(spine, (h.t - t_synch_start) / 1e3)
                # Stopping flux from the input.
                spine.ecellMan.ca_in['k'] = 0
                # Re-enabling pump and leak.
                spine.ecellMan.ca_leak['vmax'] = param['ca_leak_vmax']
                spine.ecellMan.ca_pump['vmax'] = param['ca_pump_vmax']
                update_synape_weight(spine, weight_baseline)
            t_synch_start = h.t # Resetting the t_start to the new NEURON time.

def sync_calcium(spine, dtNeuron, delta_calcium_sampling):
    """"
    Calculate the flux of the calcium in the spine_head and synch
    it with ecell.
    """
    if hasattr(spine, 'ecellMan'):

        k_ca_flux = get_calcium_flux(dtNeuron,
                                     delta_calcium_sampling,
                                     spine)
        # Unit conversion in update_calcium
        spine.update_calcium(k_ca_flux)

def get_calcium_flux(dtNeuron, delta_calcium_sampling, spine):
    """
    Retrieving the calcium in the interval. end is always -1 because is
    the last timepoint available, start is when the interval has begun
    """
    start_index = -int(delta_calcium_sampling / dtNeuron)
    # Getting the calcium value
    vec_spine_head_cai = manager.get_vector(spine.head, 'cai')
    vec_spine_head_cali = manager.get_vector(spine.head, 'cali')

    head_cai = vec_spine_head_cai.x[start_index]
    head_cali = vec_spine_head_cali.x[start_index]
    electrical_ca_start = head_cai + head_cali

    head_cai = vec_spine_head_cai.x[-1]
    head_cali = vec_spine_head_cali.x[-1]
    electrical_ca_end = head_cai + head_cali
    electrical_diff = electrical_ca_end - electrical_ca_start
    print "Len vecs: %s start_idx: %s" %(len(vec_spine_head_cali), start_index)
    print "Electrical calcium start: %s end: %s difference: %s" %(electrical_ca_start,
                                                                  electrical_ca_end,
                                                                  electrical_diff)
    # Calculating the flux
    k_calcium_flux = electrical_diff / delta_calcium_sampling

    return k_calcium_flux

def update_synape_weight(spine, baseline):
    """
    Update the electrical weight's synapses.

    spine : the spine where the weight should be updated
    baseline : the equilibrium concentration of AMPAR-P which we use as reference
    and  put equal to one.
    """

    # Updating the AMPA synapses
    for synapse in spine.synapses:
        if synapse.chan_type == 'ampa':
        # Retrieve the value of the weight.
            weight = spine.ecellMan.ampar_P['Value']/baseline
            synapse.netCon.weight[0] = weight
            # The weight of the ampa is a double list
            # Check the specs in synapse weight for more info.
            synapse.weight[0].append(h.t)
            synapse.weight[1].append(weight)
            print "Updating synapse weight in %s, time [ms]: %s, weight: %s" %(spine.id,
                                                                               h.t,
                                                                               weight)

def create_excitatory_inputs(stim_spines_id, neuron_time_interval, param):
    """
    Create the excitatory inputs according to the parametes file.

    Create the NEURON inputs on each synapses according to the parameters.
    Set and initialize the ecell biochemical simulator in the stimulated
    spine."""

    excitatory_stimuli = []

    for spine_id in stim_spines_id:
        if spine_id in param.keys():
            spine = nrnSim.spines[spine_id]
            for stim_id in param[spine.id]:
                stim_dictionary = param[stim_id]
                stim = Stimul((stim_dictionary['t_stim']* 1e3),
                              stim_dictionary['numbers'],
                              stim_dictionary['delay'],
                              stim_dictionary['type'])
                if stim.chan_type == 'ampa':
                    for syn in spine.synapses:
                        if syn.chan_type == 'ampa':
                            syn.stims.append(stim)
                elif stim.chan_type == 'nmda':# more than one stim
                    for syn in spine.synapses:
                        if syn.chan_type == 'nmda':
                            syn.stims.append(stim)

                stims_time = stim.get_stims_time()
                excitatory_stimuli.extend(stims_time)

            spine.deploy_stims(neuron_time_interval)
            spine.setupBioSim() # Initializing ecell

    excitatory_stimuli = list(set(excitatory_stimuli))
    excitatory_stimuli.sort()
    return excitatory_stimuli

def advance_quickly(tmp_tstop, stim_spines_id, weight_baseline):
    """
    Advance the two simulators quickly in an independent way. Synapse weight
    is synchronized at the end
    """
    #Update the weight
    for spine_id in stim_spines_id:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

    delta_ecell = tmp_tstop - h.t
    delta_ecell_seconds = delta_ecell / 1e3
    print ("\nAdvance quickly routine.")
    print ("Current Neuron time: %s, aimed tstop[ms]: %s") %(h.t, tmp_tstop)
    print ("Delta applied on Ecell simulator [s]: %s\n") % delta_ecell_seconds
    nrnSim.run(tmp_tstop)
    for spine_id in stim_spines_id:
        spine = nrnSim.spines[spine_id]
        advance_ecell(spine, delta_ecell_seconds)
        update_synape_weight(spine, weight_baseline)

def run_simulation(tStop_final, t_buffer, dtNeuron,
                   delta_calcium_sampling, weight_baseline, param):
    """
    Run the simulation. If input synchronizes the two simulators,
    otherwise run each on its own and advance quickly

    """

    # Getting the calcium before the stims
    for spine_id in param['stimulated_spines']:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

    while h.t < tStop_final:

        if excitatory_stims:
            t_stim = excitatory_stims.pop(0)
            print "Current Neuron time: %s. Current t_stim: %s, remaining input: %s" %(h.t,
                                                                                t_stim,
                                                                                len(excitatory_stims))
            if h.t < t_stim:
                advance_quickly(t_stim, stim_spines_id, weight_baseline)
                tmp_tstop = t_stim + t_buffer
                synch_simulators(tmp_tstop, stim_spines_id,
                                 dtNeuron,
                                 delta_calcium_sampling,
                                 weight_baseline)

        else:
            print "No excitatory input remaining. Quickly to the end"
            advance_quickly(tStop_final, stim_spines_id, weight_baseline)
            h.fadvance() # This is to force the latest step and avoid the infinite loop.

    # Recording last
    for spine_id in param['stimulated_spines']:
        spine = nrnSim.spines[spine_id]
        update_synape_weight(spine, weight_baseline)

def main(argv):

    if len(argv) != 2:
        print("No parameter file supplied. Abort.")
        usage = 'python spineIntegration.py parameters_file.param'
        print usage
        sys.exit()

    parameter_file = argv[1]
    param = parameters.ParameterSet(parameter_file)

    # Processing the options
    h.dt = param['dtNeuron']
    delta_calcium_sampling = param['delta_calcium_sampling']
    t_equilibrium_neuron = param['tEquilibrium_neuron']
    t_equilibrium_ecell = param['tEquilibrium_ecell']

    tStop_final = param['tStop']

    print("Starting Spine integration")

    hoc_path = "hoc"
    mod_path="mod"

    neuronsim = neuronControl.NeuronSim(mod_path=mod_path, hoc_path=hoc_path,
                              spines_dist=param['spines_dist'],
                              biochemical_filename=param['biochemical_filename'])

    kir_gkbar(param['kir_gkbar'])

    #--------------------------------------------------------------------------
    # Recording and stimul

    # Create Manager
    manager = Manager()

    # Set the stimuls to the synapses

    stim_spines_id = param['stimulated_spines']
    excitatory_stims = create_excitatory_inputs(stim_spines_id,
                                                param['time_resolution_neuron'],
                                                param)
    print "This are the time of the stims: %s" %excitatory_stims

    # Recording the sections

    variables_to_rec = param['var_to_plot']

    for var in variables_to_rec:
        for sec in h.allsec():
            manager.add_all_vecRef(var, param['time_resolution_neuron']) # Saving all the vecRef for testing
#            if sec.name() in param['section_to_plot']:
#                manager.add_vecRef(var, sec, param['time_resolution_neuron'])

    # Recording the synapses
    for spine_id in stim_spines_id:
        spine = neuronsim.spines[spine_id]
        for syn in spine.synapses:
            manager.add_synVecRef(syn)

    ##------------------------------------------------------------------------------
    ## Experiment
    neuronsim.init() # Initializing neuron

    # equilibrium
    print ("#--#")
    print ("Equilibrium run for the two simulators")
    neuronsim.run(t_equilibrium_neuron)
    for spine_id in stim_spines_id:
        advance_ecell(neuronsim.spines[spine_id], t_equilibrium_ecell)
    print ("Equilibrium run finished. Starting normal simulation.")
    print ("#--#")
    t_buffer = param['t_buffer']
    run_simulation(tStop_final, t_buffer,
                   h.dt, delta_calcium_sampling, param['weight_baseline'],
                   param)

    #------------------------------------
    # Save the Results

    # Add timeseries
    extRef = ExtRef()
    extRef.add_timeseries(manager, param['stimulated_spines'], neuronsim)

    # Saving the weight
    extRef.add_weights(manager, param['stimulated_spines'], neuronsim)

    print "Simulation Ended. Saving results"
    saving_dir = manager.create_new_dir(root='Data')
    hdf_name = 'storage.h5'
    filename = os.path.join(saving_dir, hdf_name)
    print "Results will be saved in %s" %filename
    # Saving everything
    manager.save_to_hdf(filename)

    #Let's save same plot

    for i, var in enumerate(param['var_to_plot']):
        secs = param['section_to_plot']
        vecs_to_plot = build_vecs_to_plot(var, secs, manager.refs['VecRef'])
        manager.plotVecs(vecs_to_plot, figure_num=i)

        if var == 'v':
            plt.ylabel("Voltage [mV]")
            plt.xlabel("Time [ms]")
            plt.ylim(-90) # Setting the minimum limits

        elif var == 'cai' or var == 'cali':
            plt.xlabel("Time [ms]")
            plt.ylabel("Concentration [mM]")

        elif var == 'ica':
            plt.xlabel("Time [ms]")
            plt.ylabel("Current [nA]")

        fig_file = 'plot_' + var
        plt.savefig(os.path.join(saving_dir, fig_file))

    from ecellControl import ecellManager as eM
    x_start = param['tEquilibrium_ecell']
    x_stop = x_start + param['tStop']/1e3
    for stim_spine in param['stimulated_spines']:
        spine = neuronsim.spines[stim_spine]
        eM.plotTimeCourses(spine.ecellMan.timeCourses, save=True,
                           dir=saving_dir, name=spine.id,
                           x_lims= [x_start, x_stop])
        eM.plotWeight(spine.ecellMan.timeCourses, dir=saving_dir)

if __name__ == "__main__":
    main(sys.argv)

<|endoftext|>