<commit_before>"""
Gromacs support in OpenPathSampling

OpenPathSampling supports Gromacs as an "external engine," meaning that
Gromacs itself runs as a coprocess, and OPS uses the file system as an
intermediary to monitor when a path needs to be terminated.
"""

import logging
logger = logging.getLogger(__name__)

from mdtraj.formats import TRRTrajectoryFile

from openpathsampling.engines import ExternalEngine
from openpathsampling.engines import features
from openpathsampling.engines.snapshot import BaseSnapshot
import features as gmx_features

import os
import psutil
import shlex
import numpy as np

# TODO: all gmx_features should be moved to external_md; along with the
# snapshot

@features.base.attach_features([
    features.engine,
    gmx_features.coordinates,
    gmx_features.velocities,
    gmx_features.box_vectors,
    gmx_features.file_info
])
class ExternalMDSnapshot(BaseSnapshot):
    """
    Snapshot for external MD engines

    Internally, this only stores the file_number and the file_position. All
    specific details (positions, velocities, box vectors) are loaded from
    file when requested.
    """
    def __init__(self, file_number=None, file_position=None, engine=None):
        self.file_number = file_number
        self.file_position = file_position
        self.engine = engine
        self._xyz = None
        self._velocities = None
        self._box_vectors = None

    def load_details(self):
        filename = self.engine.trajectory_filename(self.file_number)
        (xyz, vel, box) = self.engine.read_frame_data(filename,
                                                      self.file_position)
        self._xyz = xyz
        self._velocities = vel
        self._box_vectors = box

    def set_details(self, xyz, velocities, box_vectors):
        try:
            self.load_details()
        except:
            pass
        else:
            raise RuntimeError("Can't set details if frame already exists.")
        finally:
            self._xyz = xyz
            self._velocities = velocities
            self._box_vectors = box_vectors

    def clear_cache(self):
        self._xyz = None
        self._velocities = None
        self._box_vectors = None

    def __repr__(self):
        num_str = "file_number=" + str(self.file_number)
        pos_str = "file_position=" + str(self.file_position)
        eng_str = "engine=" + repr(self.engine)
        args = ", ".join([num_str, pos_str, eng_str])
        return "{cls_str}(".format(cls_str=self.cls) + args + ")"

class GromacsEngine(ExternalEngine):
    """
    External engine wrapper for Gromacs (using indirect API).

    This provides Gromacs support, using our indirect engine API (TODO
    link).

    Parameters
    ----------
    gro : string
        .gro file
    mdp : string
        .mdp file
    top : string
        .top file
    base_dir : string
        root directory where all files will be found (defaults to pwd)
    options : dict
        Dictionary of option name to value. Gromacs-specific option names
        are
            * ``gmx_prefix``: Prefix to gromacs commands, which are run as
              ``{gmx_prefix}command``. Default is 'gmx ' (note the space).
              This allows you to use either Gromacs 4 or Gromacs 5, as well
              as specifying the path to your version of Gromacs.
            * ``grompp_args``: Additional arguments to ``grompp``. The
              defaults take ``-c {self.gro} -f {self.mdp} -p {self.top}
              -t {self.input_file}``, where the input filename is set by
              :meth:`.set_filenames`. Default is the empty string.
            * ``mdrun_args``: Additional arguments to ``mdrun``. The
              defaults take ``-s topol.top -o self.output_file
              -e self.edr_file -g self.log_file``, where the ``topol.top``
              is generated by :meth:`.prepare`, and the other filenames are
              set by :meth:`.set_filenames`. Default is the empty string.
    """
    _default_options = dict(ExternalEngine._default_options,
        **{
            'gmx_executable': "gmx ",
            'grompp_args': "",
            'mdrun_args': ""
        }
    )
    def __init__(self, gro, mdp, top, options, base_dir="", prefix="gmx"):
        self.base_dir = base_dir
        self.gro = os.path.join(base_dir, gro)
        self.mdp = os.path.join(base_dir, mdp)
        self.top = os.path.join(base_dir, top)
        self.prefix = os.path.join(base_dir, prefix)

        dirs = [self.prefix + s for s in ['_trr', '_log', '_edr']]
        for d in dirs:
            try:
                os.mkdir(d)
            except OSError:
                pass  # the directory already exists

        self._file = None  # file open/close efficiency
        self._last_filename = None
        # TODO: update options with correct n_spatial, n_atoms
        # TODO: add snapshot_timestep; first via options, later read mdp
        template = None  # TODO: extract a template from the gro
        super(GromacsEngine, self).__init__(options, template,
                                             first_frame_in_file=True)

    def read_frame_data(self, filename, frame_num):
        """
        Returns pos, vel, box or raises error
        """
        if self._last_filename != filename:
            try:
                self._file.close()
            except AttributeError:
                pass  # first time thru, self._file is None
            self._file = TRRTrajectoryFile(filename)
        self._file.seek(offset=frame_num)
        data = self._file._read(n_frames=1, atom_indices=None,
                                get_velocities=True)
        return data[0][0], data[5][0], data[3][0]

    def read_frame_from_file(self, filename, frame_num):
        # note: this only needs to return the file pointers -- but should
        # only do so once that frame has been written!
        basename = os.path.basename(filename)
        # basename should be in the format [0-9]+\.trr (as set by the
        # trajectory_filename method)
        file_number = int(basename.split('.')[0])
        try:
            xyz, vel, box = self.read_frame_data(filename, frame_num)
        except IndexError:
            # this means that no such frame exists yet, so we return None
            return None
        except RuntimeError as e:
            # TODO: matches "TRR read error"
            return 'partial'
        else:
            return ExternalMDSnapshot(file_number=file_number,
                                      file_position=frame_num,
                                      engine=self)

    def write_frame_to_file(self, filename, snapshot, mode='w'):
        if os.path.isfile(filename):
            # stop if we already have this file; could also happen because
            # of a weird behavior in a mover. You must remove the files if
            # you don't want them.
            raise RuntimeError("File " + str(filename) + " exists. "
                               + "Preventing overwrite.")
        trr = TRRTrajectoryFile(filename, mode)
        # type control before passing things to Cython code
        xyz = np.asarray([snapshot.xyz], dtype=np.float32)
        time = np.asarray([0.0], dtype=np.float32)
        step = np.asarray([0], dtype=np.int32)
        box = np.asarray([snapshot.box_vectors], dtype=np.float32)
        lambd = np.asarray([0.0], dtype=np.float32)
        vel = np.asarray([snapshot.velocities], dtype=np.float32)
        trr._write(xyz, time, step, box, lambd, vel)
        trr.close()

    def trajectory_filename(self, number):
        trr_dir = self.prefix + "_trr/"
        return trr_dir + '{:07d}'.format(number) + '.trr'

    def set_filenames(self, number):
        self.input_file = os.path.join(self.base_dir, "initial_frame.trr")
        self.output_file = self.trajectory_filename(number + 1)
        num_str = '{:07d}'.format(number + 1)
        self.edr_file = os.path.join(self.prefix + "_edr", num_str + '.edr')
        self.log_file = os.path.join(self.prefix + "_log", num_str + '.log')

    def prepare(self):  # pragma: no cover
        # coverage ignored b/c Travis won't have gmx. However, we do have a
        # test that covers this if gmx is present (otherwise it is skipped)
        cmd = "gmx grompp -c {gro} -f {mdp} -p {top} -t {inp} {xtra}".format(
            gro=self.gro, mdp=self.mdp, top=self.top, inp=self.input_file,
            xtra=self.options['grompp_args']
        )
        logger.info(cmd)
        run_cmd = shlex.split(cmd)
        return_code = psutil.Popen(run_cmd, preexec_fn=os.setsid).wait()
        return return_code

    def cleanup(self):
        if os.path.isfile(self.input_file):
            os.remove(self.input_file)

    def engine_command(self):
        # gmx mdrun -s topol.tpr -o trr/0000001.trr -g 0000001.log
        args = self.options['mdrun_args'].format(prev_traj=self._traj_num-1,
                                                 next_traj=self._traj_num)
        cmd = "gmx mdrun -s topol.tpr -o {out} -e {edr} -g {log} {args}"
        cmd = cmd.format(out=self.output_file,
                         edr=self.edr_file,
                         log=self.log_file,
                         args=args)
        return cmd
<commit_msg>Add some docstrings; ExternMDSnap init calls super<commit_after>"""
Gromacs support in OpenPathSampling

OpenPathSampling supports Gromacs as an "external engine," meaning that
Gromacs itself runs as a coprocess, and OPS uses the file system as an
intermediary to monitor when a path needs to be terminated.
"""

import logging
logger = logging.getLogger(__name__)

from mdtraj.formats import TRRTrajectoryFile

from openpathsampling.engines import ExternalEngine
from openpathsampling.engines import features
from openpathsampling.engines.snapshot import BaseSnapshot
import features as gmx_features

import os
import psutil
import shlex
import numpy as np

# TODO: all gmx_features should be moved to external_md; along with the
# snapshot

@features.base.attach_features([
    features.engine,
    gmx_features.coordinates,
    gmx_features.velocities,
    gmx_features.box_vectors,
    gmx_features.file_info
])
class ExternalMDSnapshot(BaseSnapshot):
    """
    Snapshot for external MD engines

    Internally, this only stores the file_number and the file_position. All
    specific details (positions, velocities, box vectors) are loaded from
    file when requested.

    Parameters
    ----------
    file_number : int
        the number associated with the file for this snapshot; its engine
        should be able to convert this to a filename
    file_position : int
        position within the file; the engine should be able to load data for
        this specific snapshot based on this number
    engine : :class:`.DynamicsEngine`
        the engine associated with this snapshot
    """
    def __init__(self, file_number=None, file_position=None, engine=None):
        super(ExternalMDSnapshot, self).__init__()
        self.file_number = file_number
        self.file_position = file_position
        self.engine = engine
        self._xyz = None
        self._velocities = None
        self._box_vectors = None

    def load_details(self):
        """Cache coords, velocities, box vectors from the external file"""
        filename = self.engine.trajectory_filename(self.file_number)
        (xyz, vel, box) = self.engine.read_frame_data(filename,
                                                      self.file_position)
        self._xyz = xyz
        self._velocities = vel
        self._box_vectors = box

    def set_details(self, xyz, velocities, box_vectors):
        """Set coords, velocities, and box vectors.

        This is mainly used if OPS must modify/create a snapshot.

        Parameters
        ----------
        xyz : np.array
            unitless coordinates
        velocities : np.array
            velocities
        box_vectors : np.array
            unit cell for the periodic box
        """
        try:
            self.load_details()
        except:
            pass
        else:
            raise RuntimeError("Can't set details if frame already exists.")
        finally:
            self._xyz = xyz
            self._velocities = velocities
            self._box_vectors = box_vectors

    def clear_cache(self):
        """Remove internal details from snapshot.

        These details should always be accessible later using
        :method:`.load_details`. Removing them allows them memory to be
        freed.
        """
        self._xyz = None
        self._velocities = None
        self._box_vectors = None

    def __repr__(self):
        num_str = "file_number=" + str(self.file_number)
        pos_str = "file_position=" + str(self.file_position)
        eng_str = "engine=" + repr(self.engine)
        args = ", ".join([num_str, pos_str, eng_str])
        return "{cls_str}(".format(cls_str=self.cls) + args + ")"

class GromacsEngine(ExternalEngine):
    """
    External engine wrapper for Gromacs (using indirect API).

    This provides Gromacs support, using our indirect engine API (TODO
    link).

    Parameters
    ----------
    gro : string
        .gro file
    mdp : string
        .mdp file
    top : string
        .top file
    base_dir : string
        root directory where all files will be found (defaults to pwd)
    options : dict
        Dictionary of option name to value. Gromacs-specific option names
        are
            * ``gmx_prefix``: Prefix to gromacs commands, which are run as
              ``{gmx_prefix}command``. Default is 'gmx ' (note the space).
              This allows you to use either Gromacs 4 or Gromacs 5, as well
              as specifying the path to your version of Gromacs.
            * ``grompp_args``: Additional arguments to ``grompp``. The
              defaults take ``-c {self.gro} -f {self.mdp} -p {self.top}
              -t {self.input_file}``, where the input filename is set by
              :meth:`.set_filenames`. Default is the empty string.
            * ``mdrun_args``: Additional arguments to ``mdrun``. The
              defaults take ``-s topol.top -o self.output_file
              -e self.edr_file -g self.log_file``, where the ``topol.top``
              is generated by :meth:`.prepare`, and the other filenames are
              set by :meth:`.set_filenames`. Default is the empty string.
    """
    _default_options = dict(ExternalEngine._default_options,
        **{
            'gmx_executable': "gmx ",
            'grompp_args': "",
            'mdrun_args': ""
        }
    )
    def __init__(self, gro, mdp, top, options, base_dir="", prefix="gmx"):
        self.base_dir = base_dir
        self.gro = os.path.join(base_dir, gro)
        self.mdp = os.path.join(base_dir, mdp)
        self.top = os.path.join(base_dir, top)
        self.prefix = os.path.join(base_dir, prefix)

        dirs = [self.prefix + s for s in ['_trr', '_log', '_edr']]
        for d in dirs:
            try:
                os.mkdir(d)
            except OSError:
                pass  # the directory already exists

        self._file = None  # file open/close efficiency
        self._last_filename = None
        # TODO: update options with correct n_spatial, n_atoms
        # TODO: add snapshot_timestep; first via options, later read mdp
        template = None  # TODO: extract a template from the gro
        super(GromacsEngine, self).__init__(options, template,
                                             first_frame_in_file=True)

    def read_frame_data(self, filename, frame_num):
        """
        Returns pos, vel, box or raises error
        """
        if self._last_filename != filename:
            try:
                self._file.close()
            except AttributeError:
                pass  # first time thru, self._file is None
            self._file = TRRTrajectoryFile(filename)
        self._file.seek(offset=frame_num)
        data = self._file._read(n_frames=1, atom_indices=None,
                                get_velocities=True)
        return data[0][0], data[5][0], data[3][0]

    def read_frame_from_file(self, filename, frame_num):
        # note: this only needs to return the file pointers -- but should
        # only do so once that frame has been written!
        basename = os.path.basename(filename)
        # basename should be in the format [0-9]+\.trr (as set by the
        # trajectory_filename method)
        file_number = int(basename.split('.')[0])
        try:
            xyz, vel, box = self.read_frame_data(filename, frame_num)
        except IndexError:
            # this means that no such frame exists yet, so we return None
            return None
        except RuntimeError as e:
            # TODO: matches "TRR read error"
            return 'partial'
        else:
            return ExternalMDSnapshot(file_number=file_number,
                                      file_position=frame_num,
                                      engine=self)

    def write_frame_to_file(self, filename, snapshot, mode='w'):
        if os.path.isfile(filename):
            # stop if we already have this file; could also happen because
            # of a weird behavior in a mover. You must remove the files if
            # you don't want them.
            raise RuntimeError("File " + str(filename) + " exists. "
                               + "Preventing overwrite.")
        trr = TRRTrajectoryFile(filename, mode)
        # type control before passing things to Cython code
        xyz = np.asarray([snapshot.xyz], dtype=np.float32)
        time = np.asarray([0.0], dtype=np.float32)
        step = np.asarray([0], dtype=np.int32)
        box = np.asarray([snapshot.box_vectors], dtype=np.float32)
        lambd = np.asarray([0.0], dtype=np.float32)
        vel = np.asarray([snapshot.velocities], dtype=np.float32)
        trr._write(xyz, time, step, box, lambd, vel)
        trr.close()

    def trajectory_filename(self, number):
        trr_dir = self.prefix + "_trr/"
        return trr_dir + '{:07d}'.format(number) + '.trr'

    def set_filenames(self, number):
        self.input_file = os.path.join(self.base_dir, "initial_frame.trr")
        self.output_file = self.trajectory_filename(number + 1)
        num_str = '{:07d}'.format(number + 1)
        self.edr_file = os.path.join(self.prefix + "_edr", num_str + '.edr')
        self.log_file = os.path.join(self.prefix + "_log", num_str + '.log')

    @property
    def grompp_command(self):
        cmd = "gmx grompp -c {gro} -f {mdp} -p {top} -t {inp} {xtra}".format(
            gro=self.gro, mdp=self.mdp, top=self.top, inp=self.input_file,
            xtra=self.options['grompp_args']
        )

    def prepare(self):  # pragma: no cover
        # coverage ignored b/c Travis won't have gmx. However, we do have a
        # test that covers this if gmx is present (otherwise it is skipped)
        cmd = self.grompp_command
        logger.info(cmd)
        run_cmd = shlex.split(cmd)
        return_code = psutil.Popen(run_cmd, preexec_fn=os.setsid).wait()
        return return_code

    def cleanup(self):
        if os.path.isfile(self.input_file):
            os.remove(self.input_file)

    def engine_command(self):
        # gmx mdrun -s topol.tpr -o trr/0000001.trr -g 0000001.log
        args = self.options['mdrun_args'].format(prev_traj=self._traj_num-1,
                                                 next_traj=self._traj_num)
        cmd = "{gmx}mdrun -s topol.tpr -o {out} -e {edr} -g {log} {args}"
        cmd = cmd.format(gmx=self.options['gmx_executable'],
                         out=self.output_file,
                         edr=self.edr_file,
                         log=self.log_file,
                         args=args)
        return cmd
<|endoftext|>


<commit_before>from django.db import models
from django.core.exceptions import ValidationError, ObjectDoesNotExist

import mozdns
from mozdns.domain.models import Domain, _name_to_domain
from mozdns.models import MozdnsRecord
from mozdns.validation import validate_name, find_root_domain
from mozdns.search_utils import smart_fqdn_exists

import pdb

class CNAME(MozdnsRecord):
    """CNAMES can't point to an any other records. Said another way,
    CNAMES can't be at the samle level as any other record. This means
    that when you are creating a CNAME every other record type must be
    checked to make sure that the name about to be taken by the CNAME
    isn't taken by another record. Likewise, all other records must
    check that no CNAME exists with the same name before being created.

    >>> CNAME(label = label, domain = domain, target = target)

    """
    # TODO cite an RFC for that ^ (it's around somewhere)
    id = models.AutoField(primary_key=True)
    target = models.CharField(max_length=100, validators=[validate_name],
            help_text="CNAME Target")
    target_domain = models.ForeignKey(Domain, null=True,
                        related_name='target_domains', blank=True,
                        on_delete=models.SET_NULL)

    search_fields = ('fqdn', 'target')

    def details(self):
        return  (
                    ('FQDN', self.fqdn),
                    ('Record Type', 'CNAME'),
                    ('Target', self.target),
               )

    class Meta:
        db_table = 'cname'
        unique_together = ('domain', 'label', 'target')

    @classmethod
    def get_api_fields(cls):
        return super(CNAME, cls).get_api_fields() + ['target']

    def save(self, *args, **kwargs):
        # If label, and domain have not changed, don't mark our domain for
        # rebuilding.
        if self.pk:  # We need to exist in the db first.
            db_self = CNAME.objects.get(pk=self.pk)
            if db_self.label == self.label and db_self.domain == self.domain:
                kwargs['no_build'] = True
                 # Either nothing has changed or just target_domain. We want
                 # rebuild.
        super(CNAME, self).save(*args, **kwargs)

    def clean(self, *args, **kwargs):
        super(CNAME, self).clean(*args, **kwargs)
        super(CNAME, self).check_for_delegation()
        self.check_SOA_condition()
        self.target_domain = _name_to_domain(self.target)
        self.existing_node_check()

    def __str__(self):
        return "{0} CNAME {1}".format(self.fqdn, self.target)

    def check_SOA_condition(self):
        """We need to check if the domain is the root domain in a zone.
        If the domain is the root domain, it will have an soa, but the
        master domain will have no soa (or it will have a a different
        soa).
        """
        try:
            self.domain
        except ObjectDoesNotExist:
            return # Validation will fail eventually
        root_domain = find_root_domain(self.domain.soa)
        if root_domain is None:
            return
        if self.fqdn == root_domain.name:
            raise ValidationError("You cannot create a CNAME that points to a "
                                  "domain at the root of a zone.")
        return

    def existing_node_check(self):
        """Make sure no other nodes exist at the level of this CNAME.

            "If a CNAME RR is present at a node, no other data should be
            present; this ensures that the data for
            a canonical name and its aliases cannot be different."

            -- `RFC 1034 <http://tools.ietf.org/html/rfc1034>`_

        For example, this would be bad::

            FOO.BAR.COM     CNAME       BEE.BAR.COM

            BEE.BAR.COM     A           128.193.1.1

            FOO.BAR.COM     TXT         "v=spf1 include:foo.com -all"

        If you queried the ``FOO.BAR.COM`` name, the class of the record
        that would be returned would be ambiguous.

        .. note::
            The following records classes are checked.
                * :class:`AddressRecord` (A and AAAA)
                * :class:`SRV`
                * :class:`TXT`
                * :class:`MX`
        """
        qset = smart_fqdn_exists(self.fqdn, cn=False)
        if qset:
            objects = qset.all()
            raise ValidationError("Objects with this name already exist: {0}".
                                  format(objects))
        MX = mozdns.mx.models.MX
        if MX.objects.filter(server=self.fqdn):
            raise ValidationError("RFC 2181 says you shouldn't point MX "
                                    "records at CNAMEs")
<commit_msg>better error message<commit_after>from django.db import models
from django.core.exceptions import ValidationError, ObjectDoesNotExist

import mozdns
from mozdns.domain.models import Domain, _name_to_domain
from mozdns.models import MozdnsRecord
from mozdns.validation import validate_name, find_root_domain
from mozdns.search_utils import smart_fqdn_exists

import pdb

class CNAME(MozdnsRecord):
    """CNAMES can't point to an any other records. Said another way,
    CNAMES can't be at the samle level as any other record. This means
    that when you are creating a CNAME every other record type must be
    checked to make sure that the name about to be taken by the CNAME
    isn't taken by another record. Likewise, all other records must
    check that no CNAME exists with the same name before being created.

    >>> CNAME(label = label, domain = domain, target = target)

    """
    # TODO cite an RFC for that ^ (it's around somewhere)
    id = models.AutoField(primary_key=True)
    target = models.CharField(max_length=100, validators=[validate_name],
            help_text="CNAME Target")
    target_domain = models.ForeignKey(Domain, null=True,
                        related_name='target_domains', blank=True,
                        on_delete=models.SET_NULL)

    search_fields = ('fqdn', 'target')

    def details(self):
        return  (
                    ('FQDN', self.fqdn),
                    ('Record Type', 'CNAME'),
                    ('Target', self.target),
               )

    class Meta:
        db_table = 'cname'
        unique_together = ('domain', 'label', 'target')

    @classmethod
    def get_api_fields(cls):
        return super(CNAME, cls).get_api_fields() + ['target']

    def save(self, *args, **kwargs):
        # If label, and domain have not changed, don't mark our domain for
        # rebuilding.
        if self.pk:  # We need to exist in the db first.
            db_self = CNAME.objects.get(pk=self.pk)
            if db_self.label == self.label and db_self.domain == self.domain:
                kwargs['no_build'] = True
                 # Either nothing has changed or just target_domain. We want
                 # rebuild.
        super(CNAME, self).save(*args, **kwargs)

    def clean(self, *args, **kwargs):
        super(CNAME, self).clean(*args, **kwargs)
        super(CNAME, self).check_for_delegation()
        self.check_SOA_condition()
        self.target_domain = _name_to_domain(self.target)
        self.existing_node_check()

    def __str__(self):
        return "{0} CNAME {1}".format(self.fqdn, self.target)

    def check_SOA_condition(self):
        """We need to check if the domain is the root domain in a zone.
        If the domain is the root domain, it will have an soa, but the
        master domain will have no soa (or it will have a a different
        soa).
        """
        try:
            self.domain
        except ObjectDoesNotExist:
            return # Validation will fail eventually
        root_domain = find_root_domain(self.domain.soa)
        if root_domain is None:
            return
        if self.fqdn == root_domain.name:
            raise ValidationError("You cannot create a CNAME that points to "
                                  "the root of a zone.")
        return

    def existing_node_check(self):
        """Make sure no other nodes exist at the level of this CNAME.

            "If a CNAME RR is present at a node, no other data should be
            present; this ensures that the data for
            a canonical name and its aliases cannot be different."

            -- `RFC 1034 <http://tools.ietf.org/html/rfc1034>`_

        For example, this would be bad::

            FOO.BAR.COM     CNAME       BEE.BAR.COM

            BEE.BAR.COM     A           128.193.1.1

            FOO.BAR.COM     TXT         "v=spf1 include:foo.com -all"

        If you queried the ``FOO.BAR.COM`` name, the class of the record
        that would be returned would be ambiguous.

        .. note::
            The following records classes are checked.
                * :class:`AddressRecord` (A and AAAA)
                * :class:`SRV`
                * :class:`TXT`
                * :class:`MX`
        """
        qset = smart_fqdn_exists(self.fqdn, cn=False)
        if qset:
            objects = qset.all()
            raise ValidationError("Objects with this name already exist: {0}".
                                  format(objects))
        MX = mozdns.mx.models.MX
        if MX.objects.filter(server=self.fqdn):
            raise ValidationError("RFC 2181 says you shouldn't point MX "
                                    "records at CNAMEs")
<|endoftext|>


<commit_before>import blimpy
import argparse
import math
import sys

#stdDF = 2.9296875

# read input and output frequency, and output file name

parser = argparse.ArgumentParser(description='Dices hdf5 or fil files and writes to hdf5')
parser.add_argument('-f', '--input_filename', action='store', default=None, dest='in_fname', type=str, help='Name of file to write from (HDF5 or FIL)')
parser.add_argument('-b', action='store', default=None, dest='f_start', type=float, help='Start frequency in MHz')
parser.add_argument('-e', action='store', default=None, dest='f_stop', type=float, help='Stop frequency in MHz')
parser.add_argument('-o', '--output_filename', action='store', default=None, dest='out_fname', type=str, help='Name of file to write to (HDF5)')

args = parser.parse_args()

if len(sys.argv) == 1:
        print '\nindicate file name and start and stop frequencies\n'
        sys.exit()

if args.in_fname == None:
        print '\nneed to indicate input file name\n'
        sys.exit()

if args.out_fname == None:
        if args.in_fname[len(args.in_fname)-4:] == '.fil':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.fil','_diced.h5')
        if args.in_fname[len(args.in_fname)-3:] == '.h5':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.h5','_diced.h5')

if args.f_start == None or args.f_stop == None:
        print '\nstart and end frequencies must be informed\n'
        sys.exit()

# read start frequency and bandwidth from data set

file_big = blimpy.Waterfall(args.in_fname)
f_min_file = file_big.header['fch1']
f_max_file = file_big.header['fch1'] + file_big.header['nchans']*file_big.header['foff']

if f_max_file < f_min_file:
        f_max_file,f_min_file = f_min_file,f_max_file

FreqBWFile = f_max_file-f_min_file
stdDF = FreqBWFile / float(file_big.calc_n_coarse_chan())

if args.f_stop < args.f_start:
        args.f_stop,args.f_start = args.f_start,args.f_stop

if args.f_start < f_max_file and args.f_start > f_min_file and args.f_stop > f_max_file:
        args.f_stop = f_max_file
        print '\nWarning : higher frequency set to ' + str(f_max_file) + ' MHz to match file\n'

if args.f_stop < f_max_file and args.f_stop > f_min_file and args.f_start < f_min_file:
        args.f_start = f_min_file
        print '\nWarning : lower frequency set to ' + str(f_min_file) + ' MHz to match file\n'

if args.f_start < f_min_file and args.f_stop > f_max_file:
        args.f_start = f_min_file
        args.f_stop = f_max_file
        print '\nWarning : lower frequency set to ' + str(f_min_file) + ' MHz\nand higher frequency set to ' + str(f_max_file) + ' MHz to match file\n'
        # print '\nindicated frequencies include file frequency span - no need to dice\n'
        # sys.exit()

if min(args.f_start,args.f_stop) < f_min_file or max(args.f_start,args.f_stop) > f_max_file:
        print '\nbandwidth to extract must be within ' + str(f_min_file) + ' MHz and ' + str(f_max_file) + ' MHz\n'
        sys.exit()

# calculate real coarse channel begin and end freqs

f_start_real = math.floor((min(args.f_start,args.f_stop) - f_min_file)/stdDF)*stdDF + f_min_file
f_stop_real = f_max_file - math.floor((f_max_file - max(args.f_start,args.f_stop))/stdDF)*stdDF

# print
# print "true start frequency is " + str(f_start_real)
# print "true stop frequency is " + str(f_stop_real)

print '\nwriting to ' + args.out_fname + ' extacting from ' + str(f_start_real) + ' MHz to ' + str(f_stop_real) + ' MHz\n'

# create waterfall object

file_small = blimpy.Waterfall(args.in_fname, f_start = f_start_real, f_stop = f_stop_real)

# write waterfall object

file_small.write_to_hdf5(args.out_fname)<commit_msg>added author contact details<commit_after># dice.py removes frequency channels from BL FIL of HDF5 files, and outputs HDF5 with '_diced' appended to the file name
# gregory.hellbourg@berkeley.edu

import blimpy
import argparse
import math
import sys

#stdDF = 2.9296875

# read input and output frequency, and output file name

parser = argparse.ArgumentParser(description='Dices hdf5 or fil files and writes to hdf5')
parser.add_argument('-f', '--input_filename', action='store', default=None, dest='in_fname', type=str, help='Name of file to write from (HDF5 or FIL)')
parser.add_argument('-b', action='store', default=None, dest='f_start', type=float, help='Start frequency in MHz')
parser.add_argument('-e', action='store', default=None, dest='f_stop', type=float, help='Stop frequency in MHz')
parser.add_argument('-o', '--output_filename', action='store', default=None, dest='out_fname', type=str, help='Name of file to write to (HDF5)')

args = parser.parse_args()

if len(sys.argv) == 1:
        print '\nindicate file name and start and stop frequencies\n'
        sys.exit()

if args.in_fname == None:
        print '\nneed to indicate input file name\n'
        sys.exit()

if args.out_fname == None:
        if args.in_fname[len(args.in_fname)-4:] == '.fil':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.fil','_diced.h5')
        if args.in_fname[len(args.in_fname)-3:] == '.h5':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.h5','_diced.h5')

if args.f_start == None or args.f_stop == None:
        print '\nstart and end frequencies must be informed\n'
        sys.exit()

# read start frequency and bandwidth from data set

file_big = blimpy.Waterfall(args.in_fname)
f_min_file = file_big.header['fch1']
f_max_file = file_big.header['fch1'] + file_big.header['nchans']*file_big.header['foff']

if f_max_file < f_min_file:
        f_max_file,f_min_file = f_min_file,f_max_file

FreqBWFile = f_max_file-f_min_file
stdDF = FreqBWFile / float(file_big.calc_n_coarse_chan())

if args.f_stop < args.f_start:
        args.f_stop,args.f_start = args.f_start,args.f_stop

if args.f_start < f_max_file and args.f_start > f_min_file and args.f_stop > f_max_file:
        args.f_stop = f_max_file
        print '\nWarning : higher frequency set to ' + str(f_max_file) + ' MHz to match file\n'

if args.f_stop < f_max_file and args.f_stop > f_min_file and args.f_start < f_min_file:
        args.f_start = f_min_file
        print '\nWarning : lower frequency set to ' + str(f_min_file) + ' MHz to match file\n'

if args.f_start < f_min_file and args.f_stop > f_max_file:
        args.f_start = f_min_file
        args.f_stop = f_max_file
        print '\nWarning : lower frequency set to ' + str(f_min_file) + ' MHz\nand higher frequency set to ' + str(f_max_file) + ' MHz to match file\n'
        # print '\nindicated frequencies include file frequency span - no need to dice\n'
        # sys.exit()

if min(args.f_start,args.f_stop) < f_min_file or max(args.f_start,args.f_stop) > f_max_file:
        print '\nbandwidth to extract must be within ' + str(f_min_file) + ' MHz and ' + str(f_max_file) + ' MHz\n'
        sys.exit()

# calculate real coarse channel begin and end freqs

f_start_real = math.floor((min(args.f_start,args.f_stop) - f_min_file)/stdDF)*stdDF + f_min_file
f_stop_real = f_max_file - math.floor((f_max_file - max(args.f_start,args.f_stop))/stdDF)*stdDF

# print
# print "true start frequency is " + str(f_start_real)
# print "true stop frequency is " + str(f_stop_real)

print '\nwriting to ' + args.out_fname + ' extacting from ' + str(f_start_real) + ' MHz to ' + str(f_stop_real) + ' MHz\n'

# create waterfall object

file_small = blimpy.Waterfall(args.in_fname, f_start = f_start_real, f_stop = f_stop_real)

# write waterfall object

file_small.write_to_hdf5(args.out_fname)<|endoftext|>


<commit_before># Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#          Matti Hamalainen <msh@nmr.mgh.harvard.edu>
#          Denis A. Engemann <denis.engemann@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
from scipy import linalg, optimize

from .fixes import partial
from .utils import verbose, logger
from .io.constants import FIFF

# ############################################################################
# Compute EEG sphere model

def _fwd_eeg_get_multi_sphere_model_coeffs(m, n_terms):
    """Get the model depended weighting factor for n"""
    nlayer = len(m['layers'])
    if nlayer in (0, 1):
        return 1.

    # Initialize the arrays
    c1 = np.zeros(nlayer - 1)
    c2 = np.zeros(nlayer - 1)
    cr = np.zeros(nlayer - 1)
    cr_mult = np.zeros(nlayer - 1)
    for k in range(nlayer - 1):
        c1[k] = m['layers'][k]['sigma'] / m['layers'][k + 1]['sigma']
        c2[k] = c1[k] - 1.0
        cr_mult[k] = m['layers'][k]['rel_rad']
        cr[k] = cr_mult[k]
        cr_mult[k] *= cr_mult[k]

    coeffs = np.zeros(n_terms - 1)
    for n in range(1, n_terms):
        # Increment the radius coefficients
        for k in range(nlayer - 1):
            cr[k] *= cr_mult[k]

        # Multiply the matrices
        M = np.eye(2)
        n1 = n + 1.0
        for k in range(nlayer - 2, -1, -1):
            M = np.dot([[n + n1 * c1[k], n1 * c2[k] / cr[k]],
                        [n * c2[k] * cr[k], n1 + n * c1[k]]], M)
        num = n * (2.0 * n + 1.0) ** (nlayer - 1)
        coeffs[n - 1] = num / (n * M[1, 1] + n1 * M[1, 0])
    return coeffs

def _compose_linear_fitting_data(mu, u):
    # y is the data to be fitted (nterms-1 x 1)
    # M is the model matrix      (nterms-1 x nfit-1)
    for k in range(u['nterms'] - 1):
        k1 = k + 1
        mu1n = np.power(mu[0], k1)
        u['y'][k] = u['w'][k] * (u['fn'][k1] - mu1n*u['fn'][0])
        for p in range(u['nfit'] - 1):
            u['M'][k][p] = u['w'][k] * (np.power(mu[p + 1], k1) - mu1n)

def _compute_linear_parameters(mu, u):
    """Compute the best-fitting linear parameters"""
    _compose_linear_fitting_data(mu, u)
    uu, sing, vv = linalg.svd(u['M'], full_matrices=False)

    # Compute the residuals
    u['resi'] = u['y'].copy()

    vec = np.empty(u['nfit'] - 1)
    for p in range(u['nfit'] - 1):
        vec[p] = np.dot(uu[:, p], u['y'])
        for k in range(u['nterms'] - 1):
            u['resi'][k] -= uu[k, p] * vec[p]
        vec[p] = vec[p] / sing[p]

    lambda_ = np.zeros(u['nfit'])
    for p in range(u['nfit'] - 1):
        sum_ = 0.
        for q in range(u['nfit'] - 1):
            sum_ += vv[q, p] * vec[q]
        lambda_[p + 1] = sum_
    lambda_[0] = u['fn'][0] - np.sum(lambda_[1:])
    rv = np.dot(u['resi'], u['resi']) / np.dot(u['y'], u['y'])
    return rv, lambda_

def _one_step(mu, u):
    """Evaluate the residual sum of squares fit for one set of mu values"""
    if np.abs(mu).max() > 1.0:
        return 1.0

    # Compose the data for the linear fitting, compute SVD, then residuals
    _compose_linear_fitting_data(mu, u)
    u['uu'], u['sing'], u['vv'] = linalg.svd(u['M'])
    u['resi'][:] = u['y'][:]
    for p in range(u['nfit'] - 1):
        dot = np.dot(u['uu'][p], u['y'])
        for k in range(u['nterms'] - 1):
            u['resi'][k] = u['resi'][k] - u['uu'][p, k] * dot

    # Return their sum of squares
    return np.dot(u['resi'], u['resi'])

def _fwd_eeg_fit_berg_scherg(m, nterms, nfit):
    """Fit the Berg-Scherg equivalent spherical model dipole parameters"""
    assert nfit >= 2
    u = dict(y=np.zeros(nterms-1), resi=np.zeros(nterms-1),
             nfit=nfit, nterms=nterms, M=np.zeros((nterms - 1, nfit - 1)))

    # (1) Calculate the coefficients of the true expansion
    u['fn'] = _fwd_eeg_get_multi_sphere_model_coeffs(m, nterms + 1)

    # (2) Calculate the weighting
    f = (min([layer['rad'] for layer in m['layers']]) /
         max([layer['rad'] for layer in m['layers']]))

    # correct weighting
    k = np.arange(1, nterms + 1)
    u['w'] = np.sqrt((2.0 * k + 1) * (3.0 * k + 1.0) /
                     k) * np.power(f, (k - 1.0))
    u['w'][-1] = 0

    # Do the nonlinear minimization, constraining mu to the interval [-1, +1]
    mu_0 = np.random.RandomState(0).rand(nfit) * f
    fun = partial(_one_step, u=u)
    cons = []
    for ii in range(nfit):
        for val in [1., -1.]:
            cons.append({'type': 'ineq',
                         'fun': lambda x: np.array([val * x[ii] + 1.]),
                         'jac': lambda x: np.array([0.] * ii + [val] +
                                                   [0.] * (nfit - ii - 1))})
    mu = optimize.minimize(fun, mu_0, constraints=cons, method='COBYLA',
                           tol=1e-2).x

    # (6) Do the final step: calculation of the linear parameters
    rv, lambda_ = _compute_linear_parameters(mu, u)
    order = np.argsort(mu)[::-1]
    mu, lambda_ = mu[order], lambda_[order]  # sort: largest mu first

    m['mu'] = mu
    # This division takes into account the actual conductivities
    m['lambda'] = lambda_ / m['layers'][-1]['sigma']
    m['nfit'] = nfit
    return rv

@verbose
def make_sphere_model(r0=(0., 0., 0.04), eeg_radius=0.09, verbose=None):
    """Create a spherical model for forward solution calculation

    Parameters
    ----------
    r0 : array-like | instance of mne.io.meas_info.Info
        Head center to use. If Info, the head center will be calculated
        from the digitization points.
    eeg_radius : float | None
        If float, compute spherical shells for EEG using the given radius.
        If None, exclude shells.

    Returns
    -------
    bem : dict
        A spherical BEM.
    """
    from .io.meas_info import Info
    if isinstance(r0, Info):
        r0 = fit_sphere_to_headshape(r0)[0]
    sphere = dict(r0=np.array(r0), is_sphere=True,
                  coord_frame=FIFF.FIFFV_COORD_HEAD)
    sphere['layers'] = []
    if eeg_radius is not None:
        # Eventually these could be configurable...
        rads = [0.90, 0.92, 0.97, 1.0]
        sigmas = [0.33, 1.0, 0.004, 0.33]
        order = np.argsort(rads)
        layers = sphere['layers']
        for k in range(len(rads)):
            # sort layers by (relative) radius, and scale radii
            layer = dict(rad=rads[order[k]], sigma=sigmas[order[k]])
            layer['rel_rad'] = layer['rad'] = rads[k]
            layers.append(layer)

        # scale the radii
        R = layers[-1]['rad']
        rR = layers[-1]['rel_rad']
        for layer in layers:
            layer['rad'] /= R
            layer['rel_rad'] /= rR

        #
        # Setup the EEG sphere model calculations
        #

        # Scale the relative radii
        for k in range(len(rads)):
            layers[k]['rad'] = (eeg_radius * layers[k]['rel_rad'])
        rv = _fwd_eeg_fit_berg_scherg(sphere, 200, 3)
        logger.info('\nEquiv. model fitting -> RV = %g %%' % (100 * rv))
        for k in range(3):
            logger.info('mu%d = %g\tlambda%d = %g'
                        % (k + 1, sphere['mu'][k], k + 1,
                           layers[-1]['sigma'] * sphere['lambda'][k]))
        logger.info('Set up EEG sphere model with scalp radius %7.1f mm\n'
                    % (1000 * eeg_radius,))
    return sphere

# #############################################################################
# Helpers

@verbose
def fit_sphere_to_headshape(info, dig_kinds=(FIFF.FIFFV_POINT_EXTRA,),
                            verbose=None):
    """Fit a sphere to the headshape points to determine head center

    Parameters
    ----------
    info : instance of mne.io.meas_info.Info
        Measurement info.
    dig_kinds : tuple of int
        Kind of digitization points to use in the fitting. These can be
        any kind defined in io.constants.FIFF:
            FIFFV_POINT_CARDINAL
            FIFFV_POINT_HPI
            FIFFV_POINT_EEG
            FIFFV_POINT_ECG
            FIFFV_POINT_EXTRA
        Defaults to (FIFFV_POINT_EXTRA,).

    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    radius : float
        Sphere radius in mm.
    origin_head: ndarray
        Head center in head coordinates (mm).
    origin_device: ndarray
        Head center in device coordinates (mm).
    """
    # get head digization points of the specified kind
    hsp = [p['r'] for p in info['dig'] if p['kind'] in dig_kinds]

    # exclude some frontal points (nose etc.)
    hsp = [p for p in hsp if not (p[2] < 0 and p[1] > 0)]

    if len(hsp) == 0:
        raise ValueError('No head digitization points of the specified '
                         'kinds (%s) found.' % dig_kinds)

    hsp = 1e3 * np.array(hsp)

    radius, origin_head = _fit_sphere(hsp)
    # compute origin in device coordinates
    trans = info['dev_head_t']
    if trans['from'] != FIFF.FIFFV_COORD_DEVICE\
            or trans['to'] != FIFF.FIFFV_COORD_HEAD:
        raise RuntimeError('device to head transform not found')

    head_to_dev = linalg.inv(trans['trans'])
    origin_device = 1e3 * np.dot(head_to_dev,
                                 np.r_[1e-3 * origin_head, 1.0])[:3]

    logger.info('Fitted sphere: r = %0.1f mm' % radius)
    logger.info('Origin head coordinates: %0.1f %0.1f %0.1f mm' %
                (origin_head[0], origin_head[1], origin_head[2]))
    logger.info('Origin device coordinates: %0.1f %0.1f %0.1f mm' %
                (origin_device[0], origin_device[1], origin_device[2]))

    return radius, origin_head, origin_device

def _fit_sphere(points, disp=True):
    """Aux function to fit points to a sphere"""
    # initial guess for center and radius
    xradius = (np.max(points[:, 0]) - np.min(points[:, 0])) / 2
    yradius = (np.max(points[:, 1]) - np.min(points[:, 1])) / 2

    radius_init = (xradius + yradius) / 2
    center_init = np.array([0.0, 0.0, np.max(points[:, 2]) - radius_init])

    # optimization
    x0 = np.r_[center_init, radius_init]

    def cost_fun(x, points):
        return np.sum((np.sqrt(np.sum((points - x[:3]) ** 2, axis=1)) -
                      x[3]) ** 2)

    x_opt = optimize.fmin_powell(cost_fun, x0, args=(points,),
                                 disp=disp)

    origin = x_opt[:3]
    radius = x_opt[3]
    return radius, origin
<commit_msg>ENH: Better sphere calc<commit_after># Authors: Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
#          Matti Hamalainen <msh@nmr.mgh.harvard.edu>
#          Denis A. Engemann <denis.engemann@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
from scipy import linalg, optimize

from .fixes import partial
from .utils import verbose, logger
from .io.constants import FIFF
from .externals.six import string_types

# ############################################################################
# Compute EEG sphere model

def _fwd_eeg_get_multi_sphere_model_coeffs(m, n_terms):
    """Get the model depended weighting factor for n"""
    nlayer = len(m['layers'])
    if nlayer in (0, 1):
        return 1.

    # Initialize the arrays
    c1 = np.zeros(nlayer - 1)
    c2 = np.zeros(nlayer - 1)
    cr = np.zeros(nlayer - 1)
    cr_mult = np.zeros(nlayer - 1)
    for k in range(nlayer - 1):
        c1[k] = m['layers'][k]['sigma'] / m['layers'][k + 1]['sigma']
        c2[k] = c1[k] - 1.0
        cr_mult[k] = m['layers'][k]['rel_rad']
        cr[k] = cr_mult[k]
        cr_mult[k] *= cr_mult[k]

    coeffs = np.zeros(n_terms - 1)
    for n in range(1, n_terms):
        # Increment the radius coefficients
        for k in range(nlayer - 1):
            cr[k] *= cr_mult[k]

        # Multiply the matrices
        M = np.eye(2)
        n1 = n + 1.0
        for k in range(nlayer - 2, -1, -1):
            M = np.dot([[n + n1 * c1[k], n1 * c2[k] / cr[k]],
                        [n * c2[k] * cr[k], n1 + n * c1[k]]], M)
        num = n * (2.0 * n + 1.0) ** (nlayer - 1)
        coeffs[n - 1] = num / (n * M[1, 1] + n1 * M[1, 0])
    return coeffs

def _compose_linear_fitting_data(mu, u):
    # y is the data to be fitted (nterms-1 x 1)
    # M is the model matrix      (nterms-1 x nfit-1)
    for k in range(u['nterms'] - 1):
        k1 = k + 1
        mu1n = np.power(mu[0], k1)
        u['y'][k] = u['w'][k] * (u['fn'][k1] - mu1n*u['fn'][0])
        for p in range(u['nfit'] - 1):
            u['M'][k][p] = u['w'][k] * (np.power(mu[p + 1], k1) - mu1n)

def _compute_linear_parameters(mu, u):
    """Compute the best-fitting linear parameters"""
    _compose_linear_fitting_data(mu, u)
    uu, sing, vv = linalg.svd(u['M'], full_matrices=False)

    # Compute the residuals
    u['resi'] = u['y'].copy()

    vec = np.empty(u['nfit'] - 1)
    for p in range(u['nfit'] - 1):
        vec[p] = np.dot(uu[:, p], u['y'])
        for k in range(u['nterms'] - 1):
            u['resi'][k] -= uu[k, p] * vec[p]
        vec[p] = vec[p] / sing[p]

    lambda_ = np.zeros(u['nfit'])
    for p in range(u['nfit'] - 1):
        sum_ = 0.
        for q in range(u['nfit'] - 1):
            sum_ += vv[q, p] * vec[q]
        lambda_[p + 1] = sum_
    lambda_[0] = u['fn'][0] - np.sum(lambda_[1:])
    rv = np.dot(u['resi'], u['resi']) / np.dot(u['y'], u['y'])
    return rv, lambda_

def _one_step(mu, u):
    """Evaluate the residual sum of squares fit for one set of mu values"""
    if np.abs(mu).max() > 1.0:
        return 1.0

    # Compose the data for the linear fitting, compute SVD, then residuals
    _compose_linear_fitting_data(mu, u)
    u['uu'], u['sing'], u['vv'] = linalg.svd(u['M'])
    u['resi'][:] = u['y'][:]
    for p in range(u['nfit'] - 1):
        dot = np.dot(u['uu'][p], u['y'])
        for k in range(u['nterms'] - 1):
            u['resi'][k] = u['resi'][k] - u['uu'][p, k] * dot

    # Return their sum of squares
    return np.dot(u['resi'], u['resi'])

def _fwd_eeg_fit_berg_scherg(m, nterms, nfit):
    """Fit the Berg-Scherg equivalent spherical model dipole parameters"""
    assert nfit >= 2
    u = dict(y=np.zeros(nterms-1), resi=np.zeros(nterms-1),
             nfit=nfit, nterms=nterms, M=np.zeros((nterms - 1, nfit - 1)))

    # (1) Calculate the coefficients of the true expansion
    u['fn'] = _fwd_eeg_get_multi_sphere_model_coeffs(m, nterms + 1)

    # (2) Calculate the weighting
    f = (min([layer['rad'] for layer in m['layers']]) /
         max([layer['rad'] for layer in m['layers']]))

    # correct weighting
    k = np.arange(1, nterms + 1)
    u['w'] = np.sqrt((2.0 * k + 1) * (3.0 * k + 1.0) /
                     k) * np.power(f, (k - 1.0))
    u['w'][-1] = 0

    # Do the nonlinear minimization, constraining mu to the interval [-1, +1]
    mu_0 = np.random.RandomState(0).rand(nfit) * f
    fun = partial(_one_step, u=u)
    cons = []
    for ii in range(nfit):
        for val in [1., -1.]:
            cons.append({'type': 'ineq',
                         'fun': lambda x: np.array([val * x[ii] + 1.]),
                         'jac': lambda x: np.array([0.] * ii + [val] +
                                                   [0.] * (nfit - ii - 1))})
    mu = optimize.minimize(fun, mu_0, constraints=cons, method='COBYLA',
                           tol=1e-2).x

    # (6) Do the final step: calculation of the linear parameters
    rv, lambda_ = _compute_linear_parameters(mu, u)
    order = np.argsort(mu)[::-1]
    mu, lambda_ = mu[order], lambda_[order]  # sort: largest mu first

    m['mu'] = mu
    # This division takes into account the actual conductivities
    m['lambda'] = lambda_ / m['layers'][-1]['sigma']
    m['nfit'] = nfit
    return rv

@verbose
def make_sphere_model(r0=(0., 0., 0.04), head_radius=0.09, info=None,
                      verbose=None):
    """Create a spherical model for forward solution calculation

    Parameters
    ----------
    r0 : array-like | str
        Head center to use (in head coordinates). If 'auto', the head
        center will be calculated from the digitization points in info.
    head_radius : float | str | None
        If float, compute spherical shells for EEG using the given radius.
        If 'auto', estimate an approriate radius from the dig points in Info,
        If None, exclude shells.
    info : instance of mne.io.meas_info.Info | None
        Measurement info. Only needed if ``r0`` or ``head_radius`` are
        ``'auto'``.
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    bem : dict
        A spherical BEM.
    """
    for name in ('r0', 'head_radius'):
        param = locals()[name]
        if isinstance(param, string_types):
            if param != 'auto':
                raise ValueError('%s, if str, must be "auto" not "%s"'
                                 % (name, param))

    if (isinstance(r0, string_types) and r0 == 'auto') or \
       (isinstance(head_radius, string_types) and head_radius == 'auto'):
        if info is None:
            raise ValueError('Info must not be None for auto mode')
        head_radius_fit, r0_fit = fit_sphere_to_headshape(info)[:2]
        if isinstance(r0, string_types):
            r0 = r0_fit
        if isinstance(head_radius, string_types):
            head_radius = head_radius_fit
    sphere = dict(r0=np.array(r0), is_sphere=True,
                  coord_frame=FIFF.FIFFV_COORD_HEAD)
    sphere['layers'] = []
    if head_radius is not None:
        # Eventually these could be configurable...
        rads = [0.90, 0.92, 0.97, 1.0]
        sigmas = [0.33, 1.0, 0.004, 0.33]
        order = np.argsort(rads)
        layers = sphere['layers']
        for k in range(len(rads)):
            # sort layers by (relative) radius, and scale radii
            layer = dict(rad=rads[order[k]], sigma=sigmas[order[k]])
            layer['rel_rad'] = layer['rad'] = rads[k]
            layers.append(layer)

        # scale the radii
        R = layers[-1]['rad']
        rR = layers[-1]['rel_rad']
        for layer in layers:
            layer['rad'] /= R
            layer['rel_rad'] /= rR

        #
        # Setup the EEG sphere model calculations
        #

        # Scale the relative radii
        for k in range(len(rads)):
            layers[k]['rad'] = (head_radius * layers[k]['rel_rad'])
        rv = _fwd_eeg_fit_berg_scherg(sphere, 200, 3)
        logger.info('\nEquiv. model fitting -> RV = %g %%' % (100 * rv))
        for k in range(3):
            logger.info('mu%d = %g\tlambda%d = %g'
                        % (k + 1, sphere['mu'][k], k + 1,
                           layers[-1]['sigma'] * sphere['lambda'][k]))
        logger.info('Set up EEG sphere model with scalp radius %7.1f mm\n'
                    % (1000 * head_radius,))
    return sphere

# #############################################################################
# Helpers

@verbose
def fit_sphere_to_headshape(info, dig_kinds=(FIFF.FIFFV_POINT_EXTRA,),
                            verbose=None):
    """Fit a sphere to the headshape points to determine head center

    Parameters
    ----------
    info : instance of mne.io.meas_info.Info
        Measurement info.
    dig_kinds : tuple of int
        Kind of digitization points to use in the fitting. These can be
        any kind defined in io.constants.FIFF:
            FIFFV_POINT_CARDINAL
            FIFFV_POINT_HPI
            FIFFV_POINT_EEG
            FIFFV_POINT_ECG
            FIFFV_POINT_EXTRA
        Defaults to (FIFFV_POINT_EXTRA,).

    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    radius : float
        Sphere radius in mm.
    origin_head: ndarray
        Head center in head coordinates (mm).
    origin_device: ndarray
        Head center in device coordinates (mm).
    """
    # get head digization points of the specified kind
    hsp = [p['r'] for p in info['dig'] if p['kind'] in dig_kinds]

    # exclude some frontal points (nose etc.)
    hsp = [p for p in hsp if not (p[2] < 0 and p[1] > 0)]

    if len(hsp) == 0:
        raise ValueError('No head digitization points of the specified '
                         'kinds (%s) found.' % dig_kinds)

    hsp = 1e3 * np.array(hsp)

    radius, origin_head = _fit_sphere(hsp)
    # compute origin in device coordinates
    trans = info['dev_head_t']
    if trans['from'] != FIFF.FIFFV_COORD_DEVICE \
            or trans['to'] != FIFF.FIFFV_COORD_HEAD:
        raise RuntimeError('device to head transform not found')

    head_to_dev = linalg.inv(trans['trans'])
    origin_device = 1e3 * np.dot(head_to_dev,
                                 np.r_[1e-3 * origin_head, 1.0])[:3]

    logger.info('Fitted sphere: r = %0.1f mm' % radius)
    logger.info('Origin head coordinates: %0.1f %0.1f %0.1f mm' %
                (origin_head[0], origin_head[1], origin_head[2]))
    logger.info('Origin device coordinates: %0.1f %0.1f %0.1f mm' %
                (origin_device[0], origin_device[1], origin_device[2]))

    return radius, origin_head, origin_device

def _fit_sphere(points, disp=True):
    """Aux function to fit points to a sphere"""
    # initial guess for center and radius
    xradius = (np.max(points[:, 0]) - np.min(points[:, 0])) / 2.
    yradius = (np.max(points[:, 1]) - np.min(points[:, 1])) / 2.

    radius_init = (xradius + yradius) / 2
    center_init = np.array([0.0, 0.0, np.max(points[:, 2]) - radius_init])

    # optimization
    x0 = np.r_[center_init, radius_init]

    def cost_fun(x, points):
        return np.sum((np.sqrt(np.sum((points - x[:3]) ** 2, axis=1)) -
                      x[3]) ** 2)

    x_opt = optimize.fmin_powell(cost_fun, x0, args=(points,),
                                 disp=disp)

    origin = x_opt[:3]
    radius = x_opt[3]
    return radius, origin
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import os
import sys
import sqlite3
import glob
#import ipdb

from pandas import DataFrame, Series
import pandas as pd
#from sqlalchemy import create_engine

'''
Script to clean and load election data to an sqlite database from csv files.
Takes a dircetory and will process all the csv files in the directory.
Usage: load2sqlite.py [DIRECTORY] [TABLE NAME] [SQLITE DATABASE]
       [FILTER](optional)
'''

#-----------------------------------------------------------------
# Constants
# Dictionary is loaded from a csv file this is used
# for renaming columns and setting up the sqlite schemea
# Macrons and spaces are removed as they are never good as
# for field names in a database.
# Load the csv as a pandas data frame then convert it to a
# dictionary to deal with utf-8 encoding
#-----------------------------------------------------------------
headerdf = pd.read_csv('header_lookup.csv', encoding='UTF-8', header=None)
headerdf = headerdf.set_index(headerdf[0])
HEADER_DICT = headerdf.to_dict()[1]

def load_csv (file_name):
    '''
    Loads and cleans csv files into a pandas data frame
    Ready for writing out to an sqlite database.
    '''
    #-----------------------------------------------------------------
    # Get the Electorate Number
    #-----------------------------------------------------------------
    electorate_num = int(file_name.split('_')[-1].rstrip('.csv'))

    #-----------------------------------------------------------------
    # Skip first two rows and the last 18 lines of the file
    #-----------------------------------------------------------------
    df = pd.read_csv(file_name, skiprows=2, skipfooter=18, encoding='UTF-8')

    #-----------------------------------------------------------------
    # Rename the columns
    # Strip out macrons and spaces as these are no good as field names
    #-----------------------------------------------------------------
    df = df.rename(columns=HEADER_DICT)

    #-----------------------------------------------------------------
    # drop the row 'Voting places where less than 6 votes were taken'
    #-----------------------------------------------------------------
    filter = 'Voting places where less than 6 votes were taken'
    df = df[df['Voting_Place'] != filter]

    #-----------------------------------------------------------------
    # Add  a column with the electrate number
    #-----------------------------------------------------------------
    df['ElectID'] = electorate_num

    # forward fill the first column
    # exclude the last 5 rows as it does not apply
    last5rows = len(df) - 5
    df.loc[:last5rows, 'Suburb'] = df['Suburb'].fillna(method='ffill')

    # The Maori electorates have rows that just have a surburb name in them
    # and null values for the votes
    df = df.dropna(thresh=10)

    #-----------------------------------------------------------------
    # Reorder the columns so ElecID is the first coulmn
    #-----------------------------------------------------------------
    cols = df.columns.tolist()
    cols = cols[-1:] + cols[:-1]
    return df[cols]

def create_table_query(table_name, columns):
    '''
    Takes a list of columns of votes adds a primary key.
    Returns an sql query string
    '''
    int_cols = ', '.join([col + ' INTEGER' for col in columns])

    return  """
    CREATE TABLE IF NOT EXISTS """ + table_name + """
    (ID INTEGER PRIMARY KEY, ElectID INTEGER, Suburb VARCHAR(50),
    Voting_Place VARCHAR(255), """ + int_cols + """
    );
    """

def create_insert_query(table_name, df):
    '''
    Returns the query string for inserting the data into sqlite
    '''

    columns_to_insert = ','.join(df.columns)
    wildcards = ','.join(['?'] * (len(df.columns)))
    return 'INSERT INTO %s (%s) VALUES (%s);' \
                 % (table_name, columns_to_insert, wildcards)

if __name__ == '__main__':
    # Default variables
    filter = '*.csv'
    db = 'test.sqlite'
    table_name = 'party'
    dir = 'csv'

    if len(sys.argv) < 4:
        print ("Usage: load2sqlite.py [DIRECTORY] [TABLE NAME] [SQLITE DATABASE] [FILTER](optional)")
        sys.exit()
    if len(sys.argv) == 4:
        dir = sys.argv[1]
        table_name = sys.argv[2]
        db = sys.argv[3]
    elif len(sys.argv) > 4:
        dir = sys.argv[1]
        table_name = sys.argv[2]
        db = sys.argv[3]
        filter = sys.argv[4]

    csv_files = glob.glob(os.path.join(dir, filter))
    appended_data = []

    for csv_file in csv_files:
        data = load_csv(csv_file)
        appended_data.append(data)

    results = pd.concat(appended_data)

    # Orginally used sqlalchemy but no problems with primary keys
    with sqlite3.connect(db) as con:
        query = create_table_query(table_name, results.columns[3:])
        con.execute(query)
        con.commit()

        # Prepear the data from the dataframe
        output = results.itertuples(index=False)
        data = tuple(output)

        # Write it to the database
        # Possibly need to check if there is any data in the table
        # first need to tinker with the sql query.
        con.executemany(create_insert_query(table_name, results), data)
        con.commit()

<commit_msg>Adjusted the loading of the HEADER_DICT to use the header of the csv file<commit_after>#!/usr/bin/env python2

import os
import sys
import sqlite3
import glob
#import ipdb

from pandas import DataFrame, Series
import pandas as pd
#from sqlalchemy import create_engine

'''
Script to clean and load election data to an sqlite database from csv files.
Takes a dircetory and will process all the csv files in the directory.
Usage: load2sqlite.py [DIRECTORY] [TABLE NAME] [SQLITE DATABASE]
       [FILTER](optional)
'''

#-----------------------------------------------------------------
# Constants
# Dictionary is loaded from a csv file this is used
# for renaming columns and setting up the sqlite schemea
# Macrons and spaces are removed as they are never good as
# for field names in a database.
# Load the csv as a pandas data frame then convert it to a
# dictionary to deal with utf-8 encoding
#-----------------------------------------------------------------
headerdf = pd.read_csv('header_lookup.csv', encoding='UTF-8',
                       index_col='from_csv')
HEADER_DICT = headerdf.to_dict()['header_name']

def load_csv (file_name):
    '''
    Loads and cleans csv files into a pandas data frame
    Ready for writing out to an sqlite database.
    '''
    #-----------------------------------------------------------------
    # Get the Electorate Number
    #-----------------------------------------------------------------
    electorate_num = int(file_name.split('_')[-1].rstrip('.csv'))

    #-----------------------------------------------------------------
    # Skip first two rows and the last 18 lines of the file
    #-----------------------------------------------------------------
    df = pd.read_csv(file_name, skiprows=2, skipfooter=18, encoding='UTF-8',
                     engine='python')

    #-----------------------------------------------------------------
    # Rename the columns
    # Strip out macrons and spaces as these are no good as field names
    #-----------------------------------------------------------------
    df = df.rename(columns=HEADER_DICT)

    #-----------------------------------------------------------------
    # drop the row 'Voting places where less than 6 votes were taken'
    #-----------------------------------------------------------------
    filter = 'Voting places where less than 6 votes were taken'
    df = df[df['Voting_Place'] != filter]

    #-----------------------------------------------------------------
    # Add  a column with the electrate number
    #-----------------------------------------------------------------
    df['ElectID'] = electorate_num

    # forward fill the first column
    # exclude the last 5 rows as it does not apply
    last5rows = len(df) - 5
    df.loc[:last5rows, 'Suburb'] = df['Suburb'].fillna(method='ffill')

    # The Maori electorates have rows that just have a surburb name in them
    # and null values for the votes
    df = df.dropna(thresh=10)

    #-----------------------------------------------------------------
    # Reorder the columns so ElecID is the first coulmn
    #-----------------------------------------------------------------
    cols = df.columns.tolist()
    cols = cols[-1:] + cols[:-1]
    return df[cols]

def create_table_query(table_name, columns):
    '''
    Takes a list of columns of votes adds a primary key.
    Returns an sql query string
    '''
    int_cols = ', '.join([col + ' INTEGER' for col in columns])

    return  """
    CREATE TABLE IF NOT EXISTS """ + table_name + """
    (ID INTEGER PRIMARY KEY, ElectID INTEGER, Suburb VARCHAR(50),
    Voting_Place VARCHAR(255), """ + int_cols + """
    );
    """

def create_insert_query(table_name, df):
    '''
    Returns the query string for inserting the data into sqlite
    '''

    columns_to_insert = ','.join(df.columns)
    wildcards = ','.join(['?'] * (len(df.columns)))
    return 'INSERT INTO %s (%s) VALUES (%s);' \
                 % (table_name, columns_to_insert, wildcards)

if __name__ == '__main__':
    # Default variables
    filter = '*.csv'
    db = 'test.sqlite'
    table_name = 'party'
    dir = 'csv'

    if len(sys.argv) < 4:
        print ("Usage: load2sqlite.py [DIRECTORY] [TABLE NAME] [SQLITE DATABASE] [FILTER](optional)")
        sys.exit()
    if len(sys.argv) == 4:
        dir = sys.argv[1]
        table_name = sys.argv[2]
        db = sys.argv[3]
    elif len(sys.argv) > 4:
        dir = sys.argv[1]
        table_name = sys.argv[2]
        db = sys.argv[3]
        filter = sys.argv[4]

    csv_files = glob.glob(os.path.join(dir, filter))
    appended_data = []

    for csv_file in csv_files:
        data = load_csv(csv_file)
        appended_data.append(data)

    results = pd.concat(appended_data)

    # Orginally used sqlalchemy but no problems with primary keys
    with sqlite3.connect(db) as con:
        query = create_table_query(table_name, results.columns[3:])
        con.execute(query)
        con.commit()

        # Prepear the data from the dataframe
        output = results.itertuples(index=False)
        data = tuple(output)

        # Write it to the database
        # Possibly need to check if there is any data in the table
        # first need to tinker with the sql query.
        con.executemany(create_insert_query(table_name, results), data)
        con.commit()

<|endoftext|>


<commit_before># GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from pkg_resources import resource_filename
import os
import logging
import wtforms

from mediagoblin import mg_globals, messages
from mediagoblin.plugins.recaptcha import forms as captcha_forms
from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
from mediagoblin.tools import pluginapi
from mediagoblin.tools.staticdirect import PluginStatic

from recaptcha.client import captcha

_log = logging.getLogger(__name__)

PLUGIN_DIR = os.path.dirname(__file__)

def setup_plugin():
    _log.info('Setting up recaptcha...')
    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')
    if config:
        if config.get('RECAPTCHA_USE_SSL') == True:
            _log.info('reCAPTCHA is configured to use SSL.')
        else:
            _log.info('reCAPTCHA is NOT configured to use SSL.')

        if config.get('RECAPTCHA_PUBLIC_KEY') == 'domainpublickey':
            _log.warn('reCAPTCHA public key was not specified.')
        if config.get('RECAPTCHA_PRIVATE_KEY') == 'domainprivatekey':
            _log.warn('reCAPTCHA private key was not specified.')

    pluginapi.register_template_path(os.path.join(PLUGIN_DIR, 'templates'))

    pluginapi.register_template_hooks(
        {'head': 'mediagoblin/plugins/recaptcha/bits/recaptcha_extra_head.html',
         'captcha_challenge': 'mediagoblin/plugins/recaptcha/captcha_challenge.html'})

    _log.info('Done setting up recaptcha!')

def get_registration_form(request):
    return captcha_forms.RegistrationForm(request.form)

def extra_validation(register_form):
    extra_validation_passes = False

    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')

    if register_form.validate():
        recaptcha_challenge = register_form.recaptcha_challenge_field.data if 'recaptcha_challenge_field' in register_form else None
        recaptcha_response = register_form.recaptcha_response_field.data if 'recaptcha_response_field' in register_form else None
        remote_addr = register_form.recaptcha_remote_addr.data if 'recaptcha_remote_addr' in register_form else None

        _log.debug('response field is: %r', recaptcha_response)
        _log.debug('challenge field is: %r', recaptcha_challenge)
        response = captcha.submit(
            recaptcha_challenge,
            recaptcha_response,
            config.get('RECAPTCHA_PRIVATE_KEY'),
            #request.remote_addr,
            remote_addr,
            )

        #goblin = response.is_valid
        extra_validation_passes = response.is_valid
        if response.error_code:
            _log.debug('reCAPTCHA error: %r', response.error_code)

        if not extra_validation_passes:
            register_form.recaptcha_response_field.errors.append(
                _('Sorry, captcha was incorrect. Please try again.'))

    return extra_validation_passes

def no_pass_redirect():
    return 'recaptcha'

def add_to_form_context(context):
    recaptcha_protocol = ''
    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')
    if config['RECAPTCHA_USE_SSL']:
        recaptcha_protocol = 'https'
    else:
        recaptcha_protocol = 'http'
    _log.debug('Connecting to reCAPTCHA service via %r', recaptcha_protocol)

    context['captcha'] = True
    context['recaptcha_protocol'] = recaptcha_protocol
    context['recaptcha_public_key'] = config.get('RECAPTCHA_PUBLIC_KEY'),

    return context

hooks = {
    'setup': setup_plugin,
    'auth_extra_validation': extra_validation,
    'auth_get_registration_form': get_registration_form,
    #'auth_no_pass_redirect': no_pass_redirect,
    #'template_global_context': append_to_global_context,
    #'static_setup': lambda: PluginStatic(
    #    'coreplugin_recaptcha',
    #    resource_filename('mediagoblin.plugins.recaptcha', 'static'))
    ('mediagoblin.auth.register',
     'mediagoblin/auth/register.html'): add_to_form_context,
}
<commit_msg>Removed static setup<commit_after># GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from pkg_resources import resource_filename
import os
import logging
import wtforms

from mediagoblin import mg_globals, messages
from mediagoblin.plugins.recaptcha import forms as captcha_forms
from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
from mediagoblin.tools import pluginapi

from recaptcha.client import captcha

_log = logging.getLogger(__name__)

PLUGIN_DIR = os.path.dirname(__file__)

def setup_plugin():
    _log.info('Setting up recaptcha...')
    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')
    if config:
        if config.get('RECAPTCHA_USE_SSL') == True:
            _log.info('reCAPTCHA is configured to use SSL.')
        else:
            _log.info('reCAPTCHA is NOT configured to use SSL.')

        if config.get('RECAPTCHA_PUBLIC_KEY') == 'domainpublickey':
            _log.warn('reCAPTCHA public key was not specified.')
        if config.get('RECAPTCHA_PRIVATE_KEY') == 'domainprivatekey':
            _log.warn('reCAPTCHA private key was not specified.')

    pluginapi.register_template_path(os.path.join(PLUGIN_DIR, 'templates'))

    pluginapi.register_template_hooks(
        {'head': 'mediagoblin/plugins/recaptcha/bits/recaptcha_extra_head.html',
         'captcha_challenge': 'mediagoblin/plugins/recaptcha/captcha_challenge.html'})

    _log.info('Done setting up recaptcha!')

def get_registration_form(request):
    return captcha_forms.RegistrationForm(request.form)

def extra_validation(register_form):
    extra_validation_passes = False

    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')

    if register_form.validate():
        recaptcha_challenge = register_form.recaptcha_challenge_field.data if 'recaptcha_challenge_field' in register_form else None
        recaptcha_response = register_form.recaptcha_response_field.data if 'recaptcha_response_field' in register_form else None
        remote_addr = register_form.recaptcha_remote_addr.data if 'recaptcha_remote_addr' in register_form else None

        _log.debug('response field is: %r', recaptcha_response)
        _log.debug('challenge field is: %r', recaptcha_challenge)
        response = captcha.submit(
            recaptcha_challenge,
            recaptcha_response,
            config.get('RECAPTCHA_PRIVATE_KEY'),
            #request.remote_addr,
            remote_addr,
            )

        #goblin = response.is_valid
        extra_validation_passes = response.is_valid
        if response.error_code:
            _log.debug('reCAPTCHA error: %r', response.error_code)

        if not extra_validation_passes:
            register_form.recaptcha_response_field.errors.append(
                _('Sorry, captcha was incorrect. Please try again.'))

    return extra_validation_passes

def no_pass_redirect():
    return 'recaptcha'

def add_to_form_context(context):
    recaptcha_protocol = ''
    config = pluginapi.get_config('mediagoblin.plugins.recaptcha')
    if config['RECAPTCHA_USE_SSL']:
        recaptcha_protocol = 'https'
    else:
        recaptcha_protocol = 'http'
    _log.debug('Connecting to reCAPTCHA service via %r', recaptcha_protocol)

    context['captcha'] = True
    context['recaptcha_protocol'] = recaptcha_protocol
    context['recaptcha_public_key'] = config.get('RECAPTCHA_PUBLIC_KEY'),

    return context

hooks = {
    'setup': setup_plugin,
    'auth_extra_validation': extra_validation,
    'auth_get_registration_form': get_registration_form,
    #'auth_no_pass_redirect': no_pass_redirect,
    #'template_global_context': append_to_global_context,
    ('mediagoblin.auth.register',
     'mediagoblin/auth/register.html'): add_to_form_context,
}
<|endoftext|>


<commit_before><commit_msg>remove camera file<commit_after><|endoftext|>


<commit_before>#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Implements the suite of GMPEs used for the 2018 German National Seismic Hazard
Model of Gruenthal et al. (2018)

"Gruenthal, G., Strometer, D., Bosse, C., Cotton, F. and Bindi, D. (2018)
The probabilistic seismic hazard assessment of Germany - version 2016,
considering the range or epistemic uncertainties and aleatory variability."
Bulletin of Earthquake Engineering, 16(10), 4339-4395

Module Exports :class: `AkkarEtAlRhyp2014LSD`,
                       `AkkarEtAlRhyp2014MSD`,
                       `AkkarEtAlRhyp2014HSD`,
                       `BindiEtAl2014RhypLSD`,
                       `BindiEtAl2014RhypMSD`,
                       `BindiEtAl2014RhypHSD`,
                       `CauzziEtAl2014Rhypo`,
                       `CauzziEtAl2014RhypoLSD`,
                       `CauzziEtAl2014RhypoMSD`,
                       `CauzziEtAl2014RhypoHSD`,
                       `DerrasEtAl2014Rhypo`,
                       `DerrasEtAl2014RhypoLSD`,
                       `DerrasEtAl2014RhypoMSD`,
                       `DerrasEtAl2014RhypoHSD`,
                       `BindiEtAl2017RhypoLSD,
                       `BindiEtAl2017RhypoMSD,
                       `BindiEtAl2017RhypoHSD
"""
import numpy as np
from scipy.constants import g
from openquake.hazardlib.imt import PGA, SA
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRhyp2014
from openquake.hazardlib.gsim.bindi_2014 import BindiEtAl2014Rhyp
from openquake.hazardlib.gsim.cauzzi_2014 import CauzziEtAl2014
from openquake.hazardlib.gsim.derras_2014 import DerrasEtAl2014
from openquake.hazardlib.gsim.bindi_2017 import BindiEtAl2017Rhypo

STRESS_DROP_ADJUST = {
    "L": np.log(0.75),
    "M": np.log(1.25),
    "H": np.log(1.5)}

class AkkarEtAlRhyp2014Germany(AkkarEtAlRhyp2014):
    """
    Akkar et al. (2014) "Rhypo" model with a scalar adjustment factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Bindi et al 2014
class BindiEtAl2014RhypGermany(BindiEtAl2014Rhyp):
    """
    Bindi et al. (2014) "Rhypo" model with a scalar adjustment factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

def rhypo_to_rrup(rhypo, mag):
    """
    Converts hypocentral distance to an equivalent rupture distance
    dependent on the magnitude
    """
    rrup = rhypo - (0.7108 + 2.496E-6 * (mag ** 7.982))
    rrup[rrup < 3.0] = 3.0
    return rrup

def rhypo_to_rjb(rhypo, mag):
    """
    Converts hypocentral distance to an equivalent Joyner-Boore distance
    dependent on the magnitude
    """
    epsilon = rhypo - (4.853 + 1.347E-6 * (mag ** 8.163))
    rjb = np.zeros_like(rhypo)
    idx = epsilon >= 3.
    rjb[idx] = np.sqrt((epsilon[idx] ** 2.) - 9.0)
    rjb[rjb < 0.0] = 0.0
    return rjb

# Cauzzi et al. 2014 - Converted from Rhypo
class CauzziEtAl2014Rhypo(CauzziEtAl2014):
    """
    Implements the Cauzzi et al. (2015) GMPE applying the rhypo to rrup
    adjustment factor adopted for Germany
    """
    REQUIRES_DISTANCES = set(("rhypo", ))

    def _compute_mean(self, C, rup, dists, sites, imt):
        """
        Returns the mean ground motion acceleration and velocity
        """
        # Convert rhypo to rrup
        rrup = rhypo_to_rrup(dists.rhypo, rup.mag)
        mean = (self._get_magnitude_scaling_term(C, rup.mag) +
                self._get_distance_scaling_term(C, rup.mag, rrup) +
                self._get_style_of_faulting_term(C, rup.rake) +
                self._get_site_amplification_term(C, sites.vs30))
        # convert from cm/s**2 to g for SA and from cm/s**2 to g for PGA (PGV
        # is already in cm/s) and also convert from base 10 to base e.
        if isinstance(imt, PGA):
            mean = np.log((10 ** mean) * ((2 * np.pi / 0.01) ** 2) *
                          1e-2 / g)
        elif isinstance(imt, SA):
            mean = np.log((10 ** mean) * ((2 * np.pi / imt.period) ** 2) *
                          1e-2 / g)
        else:
            mean = np.log(10 ** mean)

        return mean

class CauzziEtAl2014RhypoGermany(CauzziEtAl2014Rhypo):
    """
    Cauzzi et al. (2014) model adapted for rhypo, with the scalar adjustment
    factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Derras et al 2014
class DerrasEtAl2014Rhypo(DerrasEtAl2014):
    """
    Re-calibration of the Derras et al. (2014) GMPE taking hypocentral
    distance as an input and converting to Rjb
    """
    #: The required distance parameter is hypocentral distance
    REQUIRES_DISTANCES = set(('rhypo', ))

    def get_mean(self, C, rup, sites, dists):
        """
        Returns the mean ground motion in terms of log10 m/s/s, implementing
        equation 2 (page 502)
        """
        # W2 needs to be a 1 by 5 matrix (not a vector
        w_2 = np.array([
            [C["W_21"], C["W_22"], C["W_23"], C["W_24"], C["W_25"]]
            ])
        # Gets the style of faulting dummy variable
        sof = self._get_sof_dummy_variable(rup.rake)
        # Get the normalised coefficients
        p_n = self.get_pn(rup, sites, dists, sof)
        mean = np.zeros_like(dists.rhypo)
        # Need to loop over sites - maybe this can be improved in future?
        # ndenumerate is used to allow for application to 2-D arrays
        for idx, rval in np.ndenumerate(p_n[0]):
            # Place normalised coefficients into a single array
            p_n_i = np.array([rval, p_n[1], p_n[2][idx], p_n[3], p_n[4]])
            # Executes the main ANN model
            mean_i = np.dot(w_2, np.tanh(np.dot(self.W_1, p_n_i) + self.B_1))
            mean[idx] = (0.5 * (mean_i + C["B_2"] + 1.0) *
                         (C["tmax"] - C["tmin"])) + C["tmin"]
        return mean

    def get_pn(self, rup, sites, dists, sof):
        """
        Normalise the input parameters within their upper and lower
        defined range
        """
        # List must be in following order
        p_n = []
        # Rjb
        # Note that Rjb must be clipped at 0.1 km
        rjb = rhypo_to_rjb(dists.rhypo, rup.mag)
        rjb[rjb < 0.1] = 0.1
        p_n.append(self._get_normalised_term(np.log10(rjb),
                                             self.CONSTANTS["logMaxR"],
                                             self.CONSTANTS["logMinR"]))
        # Magnitude
        p_n.append(self._get_normalised_term(rup.mag,
                                             self.CONSTANTS["maxMw"],
                                             self.CONSTANTS["minMw"]))
        # Vs30
        p_n.append(self._get_normalised_term(np.log10(sites.vs30),
                                             self.CONSTANTS["logMaxVs30"],
                                             self.CONSTANTS["logMinVs30"]))
        # Depth
        p_n.append(self._get_normalised_term(rup.hypo_depth,
                                             self.CONSTANTS["maxD"],
                                             self.CONSTANTS["minD"]))
        # Style of Faulting
        p_n.append(self._get_normalised_term(sof,
                                             self.CONSTANTS["maxFM"],
                                             self.CONSTANTS["minFM"]))
        return p_n

class DerrasEtAl2014RhypoGermany(DerrasEtAl2014Rhypo):
    """
    Derras et al. (2014) with Rhypo adjustment plus scalar adjustment factor
    for Germany
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Bindi et al. (2017)
class BindiEtAl2017RhypoGermany(BindiEtAl2017Rhypo):
    """
    Bindi et al. 2017 with scalar stress drop adjustment factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs
<commit_msg>Minor pep8 fixes<commit_after>#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Implements the suite of GMPEs used for the 2018 German National Seismic Hazard
Model of Gruenthal et al. (2018)

"Gruenthal, G., Strometer, D., Bosse, C., Cotton, F. and Bindi, D. (2018)
The probabilistic seismic hazard assessment of Germany - version 2016,
considering the range or epistemic uncertainties and aleatory variability."
Bulletin of Earthquake Engineering, 16(10), 4339-4395

Module Exports :class: `AkkarEtAlRhyp2014Germany`,
                       `BindiEtAl2014RhypGermany`,
                       `CauzziEtAl2014Rhypo`,
                       `CauzziEtAl2014RhypoGermany`,
                       `DerrasEtAl2014Rhypo`,
                       `DerrasEtAl2014RhypoGermany`,
                       `BindiEtAl2017RhypoGermany`
"""
import numpy as np
from scipy.constants import g
from openquake.hazardlib.imt import PGA, SA
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRhyp2014
from openquake.hazardlib.gsim.bindi_2014 import BindiEtAl2014Rhyp
from openquake.hazardlib.gsim.cauzzi_2014 import CauzziEtAl2014
from openquake.hazardlib.gsim.derras_2014 import DerrasEtAl2014
from openquake.hazardlib.gsim.bindi_2017 import BindiEtAl2017Rhypo

class AkkarEtAlRhyp2014Germany(AkkarEtAlRhyp2014):
    """
    Akkar et al. (2014) "Rhypo" model with a scalar adjustment factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Bindi et al 2014
class BindiEtAl2014RhypGermany(BindiEtAl2014Rhyp):
    """
    Bindi et al. (2014) "Rhypo" model with a scalar adjustment factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

def rhypo_to_rrup(rhypo, mag):
    """
    Converts hypocentral distance to an equivalent rupture distance
    dependent on the magnitude
    """
    rrup = rhypo - (0.7108 + 2.496E-6 * (mag ** 7.982))
    rrup[rrup < 3.0] = 3.0
    return rrup

def rhypo_to_rjb(rhypo, mag):
    """
    Converts hypocentral distance to an equivalent Joyner-Boore distance
    dependent on the magnitude
    """
    epsilon = rhypo - (4.853 + 1.347E-6 * (mag ** 8.163))
    rjb = np.zeros_like(rhypo)
    idx = epsilon >= 3.
    rjb[idx] = np.sqrt((epsilon[idx] ** 2.) - 9.0)
    rjb[rjb < 0.0] = 0.0
    return rjb

# Cauzzi et al. 2014 - Converted from Rhypo
class CauzziEtAl2014Rhypo(CauzziEtAl2014):
    """
    Implements the Cauzzi et al. (2015) GMPE applying the rhypo to rrup
    adjustment factor adopted for Germany
    """
    REQUIRES_DISTANCES = set(("rhypo", ))

    def _compute_mean(self, C, rup, dists, sites, imt):
        """
        Returns the mean ground motion acceleration and velocity
        """
        # Convert rhypo to rrup
        rrup = rhypo_to_rrup(dists.rhypo, rup.mag)
        mean = (self._get_magnitude_scaling_term(C, rup.mag) +
                self._get_distance_scaling_term(C, rup.mag, rrup) +
                self._get_style_of_faulting_term(C, rup.rake) +
                self._get_site_amplification_term(C, sites.vs30))
        # convert from cm/s**2 to g for SA and from cm/s**2 to g for PGA (PGV
        # is already in cm/s) and also convert from base 10 to base e.
        if isinstance(imt, PGA):
            mean = np.log((10 ** mean) * ((2 * np.pi / 0.01) ** 2) *
                          1e-2 / g)
        elif isinstance(imt, SA):
            mean = np.log((10 ** mean) * ((2 * np.pi / imt.period) ** 2) *
                          1e-2 / g)
        else:
            mean = np.log(10 ** mean)

        return mean

class CauzziEtAl2014RhypoGermany(CauzziEtAl2014Rhypo):
    """
    Cauzzi et al. (2014) model adapted for rhypo, with the scalar adjustment
    factor
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Derras et al 2014
class DerrasEtAl2014Rhypo(DerrasEtAl2014):
    """
    Re-calibration of the Derras et al. (2014) GMPE taking hypocentral
    distance as an input and converting to Rjb
    """
    #: The required distance parameter is hypocentral distance
    REQUIRES_DISTANCES = set(('rhypo', ))

    def get_mean(self, C, rup, sites, dists):
        """
        Returns the mean ground motion in terms of log10 m/s/s, implementing
        equation 2 (page 502)
        """
        # W2 needs to be a 1 by 5 matrix (not a vector
        w_2 = np.array([
            [C["W_21"], C["W_22"], C["W_23"], C["W_24"], C["W_25"]]
            ])
        # Gets the style of faulting dummy variable
        sof = self._get_sof_dummy_variable(rup.rake)
        # Get the normalised coefficients
        p_n = self.get_pn(rup, sites, dists, sof)
        mean = np.zeros_like(dists.rhypo)
        # Need to loop over sites - maybe this can be improved in future?
        # ndenumerate is used to allow for application to 2-D arrays
        for idx, rval in np.ndenumerate(p_n[0]):
            # Place normalised coefficients into a single array
            p_n_i = np.array([rval, p_n[1], p_n[2][idx], p_n[3], p_n[4]])
            # Executes the main ANN model
            mean_i = np.dot(w_2, np.tanh(np.dot(self.W_1, p_n_i) + self.B_1))
            mean[idx] = (0.5 * (mean_i + C["B_2"] + 1.0) *
                         (C["tmax"] - C["tmin"])) + C["tmin"]
        return mean

    def get_pn(self, rup, sites, dists, sof):
        """
        Normalise the input parameters within their upper and lower
        defined range
        """
        # List must be in following order
        p_n = []
        # Rjb
        # Note that Rjb must be clipped at 0.1 km
        rjb = rhypo_to_rjb(dists.rhypo, rup.mag)
        rjb[rjb < 0.1] = 0.1
        p_n.append(self._get_normalised_term(np.log10(rjb),
                                             self.CONSTANTS["logMaxR"],
                                             self.CONSTANTS["logMinR"]))
        # Magnitude
        p_n.append(self._get_normalised_term(rup.mag,
                                             self.CONSTANTS["maxMw"],
                                             self.CONSTANTS["minMw"]))
        # Vs30
        p_n.append(self._get_normalised_term(np.log10(sites.vs30),
                                             self.CONSTANTS["logMaxVs30"],
                                             self.CONSTANTS["logMinVs30"]))
        # Depth
        p_n.append(self._get_normalised_term(rup.hypo_depth,
                                             self.CONSTANTS["maxD"],
                                             self.CONSTANTS["minD"]))
        # Style of Faulting
        p_n.append(self._get_normalised_term(sof,
                                             self.CONSTANTS["maxFM"],
                                             self.CONSTANTS["minFM"]))
        return p_n

class DerrasEtAl2014RhypoGermany(DerrasEtAl2014Rhypo):
    """
    Derras et al. (2014) with Rhypo adjustment plus scalar adjustment factor
    for Germany
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs

# Bindi et al. (2017)
class BindiEtAl2017RhypoGermany(BindiEtAl2017Rhypo):
    """
    Bindi et al. 2017 with scalar stress drop adjustment factor for Germany
    """
    def __init__(self, adjustment_factor="1.0"):
        super().__init__()
        self.adjustment_factor = np.log(float(adjustment_factor))

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the adjusted mean and standard deviation
        """
        mean, stddevs = super().get_mean_and_stddevs(sites, rup, dists, imt,
                                                     stddev_types)
        return mean + self.adjustment_factor, stddevs
<|endoftext|>