<commit_before>""""""

from copy import deepcopy
import os
import posixpath

import jinja2

from neurodocker.utils import load_yaml

GENERIC_VERSION = 'generic'

apt_install = """apt-get update -qq
apt-get install -y {{ apt_opts|default('-q --no-install-recommends', false) }} \\\
{% for pkg in pkgs %}
    {% if not loop.last -%}
    {{ pkg }} \\\
    {%- else -%}
    {{ pkg }}
    {%- endif -%}
{% endfor %}
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
"""
apt_install = jinja2.Template(apt_install)

yum_install = """yum install -y {{ yum_opts|default('-q', true) }} \\\
{% for pkg in pkgs %}
    {% if not loop.last -%}
    {{ pkg }} \\\
    {%- else -%}
    {{ pkg }}
    {%- endif -%}
{% endfor %}
yum clean packages
rm -rf /var/cache/yum/* /tmp/* /var/tmp/*
"""
yum_install = jinja2.Template(yum_install)

deb_install = """{% for deb_url in debs -%}
curl -sSL --retry 5 -o /tmp/toinstall.deb {{ deb_url }}
dpkg -i /tmp/toinstall.deb
rm /tmp/toinstall.deb
{% endfor -%}
apt-get install -f
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
"""
deb_install = jinja2.Template(deb_install)

def _load_global_specs():

    def load_global_specs(glob_pattern):
        import glob

        def load_interface_spec(filepath):
            _, filename = os.path.split(filepath)
            key, _ = os.path.splitext(filename)
            return key, load_yaml(filepath)

        interface_yamls = glob.glob(glob_pattern)
        instructions = {}
        for ff in interface_yamls:
            key, data = load_interface_spec(ff)
            instructions[key] = data
        return instructions

    base_path = os.path.dirname(os.path.realpath(__file__))
    glob_pattern = os.path.join(base_path, '..', 'templates', '*.yaml')
    return load_global_specs(glob_pattern)

_global_specs = _load_global_specs()

def _interface_exists_in_yaml(name):
    return name in _global_specs.keys()

class _Resolver:
    """
    Parameters
    ----------
    d : dict
    """
    def __init__(self, d):
        self._d = d
        self._generic_only = self._d.keys() == {GENERIC_VERSION}

    @property
    def versions(self):
        return set(self._d.keys())

    def version_exists(self, version):
        if self._generic_only:
            return True
        else:
            return version in self.versions

    def check_version_exists(self, version):
        if not self.version_exists(version):
            raise ValueError("version '{}' not found.".format(version))

    def get_version_key(self, version):
        """Return version key to use given a specific version. For example,
        if a dictionary only has instructions for version 'generic', will
        return 'generic' given any version string.

        Raises
        ------
        `ValueError` if no key could be found for requested version.
        """
        if self._generic_only:
            return GENERIC_VERSION
        else:
            self.check_version_exists(version)
            return version

    def version_has_method(self, version, method):
        version_key = self.get_version_key(version)
        return method in self._d[version_key].keys()

    def check_version_has_method(self, version, method):
        if not self.version_has_method(version, method):
            raise ValueError(
                "version '{}' does not have method '{}'"
                .format(version, method)
            )

    def version_method_has_instructions(self, version, method):
        version_key = self.get_version_key(version)
        self.check_version_has_method(version_key, method)
        return 'instructions' in self._d[version_key][method].keys()

    def check_version_method_has_instructions(self, version, method):
        if not self.version_method_has_instructions(version, method):
            raise ValueError(
                "installation method '{}' for version '{}' does not have an"
                " 'instructions' key.".format(method, version)
            )

    def binaries_has_url(self, version):
        version_key = self.get_version_key(version)
        if self.version_has_method(version_key, 'binaries'):
            try:
                urls = self._d[version_key]['binaries']['urls'].keys()
                return version in urls
            except KeyError:
                raise ValueError(
                    "no binary URLs defined for version '{}'".format(version)
                )
        else:
            raise ValueError(
                "no binary installation method defined for version '{}"
                .format(version)
            )

    def check_binaries_has_url(self, version):
        if not self.binaries_has_url(version):
            version_key = self.get_version_key(version)
            valid_vers = self._d[version_key]['binaries']['urls']
            raise ValueError(
                "URL not found for version '{}'. Valid versions are {}"
                .format(version, ', '.join(valid_vers))
            )

    def binaries_url(self, version):
        self.check_binaries_has_url(version)
        version_key = self.get_version_key(version)
        return self._d[version_key]['binaries']['urls'][version]

class _BaseInterface:
    """Base class for interface objects."""

    def __init__(self, name, version, pkg_manager, method='binaries',
                 install_path=None, **kwargs):
        self._name = name
        self._version = version
        self._pkg_manager = pkg_manager
        self._method = method
        self._install_path = install_path
        self.__dict__.update(**kwargs)

        if not _interface_exists_in_yaml(self._name):
            raise ValueError(
                "No YAML entry for package '{}'".format(self._name)
            )
        self._resolver = _Resolver(_global_specs[self._name])

        self._version_key = self._resolver.get_version_key(self._version)
        self._resolver.check_version_exists(self._version)
        self._resolver.check_version_has_method(self._version, self._method)
        self._resolver.check_version_method_has_instructions(
            self._version, self._method
        )

        if method == 'binaries':
            self.binaries_url = self._resolver.binaries_url(self._version)

        self._instance_specs = deepcopy(
            _global_specs[self._name][self._version_key][self._method]
        )

        self._run = self._instance_specs['instructions']
        self._dependencies = self._get_dependencies()

        self._env = self._instance_specs.get('env', None)

        # Set default curl options for all interfaces.
        self.__dict__.setdefault("curl_opts", "-fsSL --retry 5")

    @property
    def _pretty_name(self):
        return self.__class__.__name__

    @property
    def name(self):
        return self._name

    @property
    def version(self):
        return self._version

    @property
    def versions(self):
        return self._resolver.versions

    @property
    def pkg_manager(self):
        return self._pkg_manager

    @property
    def method(self):
        return self._method

    @property
    def env(self):
        return self._env

    @property
    def run(self):
        return self._run

    @property
    def install_path(self):
        if self._install_path is None:
            path = posixpath.join(posixpath.sep, 'opt', '{}-{}')
            return path.format(self._name, self._version)
        return self._install_path

    @property
    def dependencies(self):
        return self._dependencies

    def _get_dependencies(self):
        if 'dependencies' not in self._instance_specs.keys():
            return None
        if self._instance_specs['dependencies'] is None:
            return None
        try:
            deps = self._instance_specs['dependencies'][self._pkg_manager]
            return deps.split() if deps else None
        except KeyError:
            return None

    def _get_debs(self):
        if 'dependencies' not in self._instance_specs.keys():
            return None
        if self._instance_specs['dependencies'] is None:
            return None
        try:
            debs = self._instance_specs['dependencies']['debs']
            return debs if debs else None
        except KeyError:
            return None

    def install_dependencies(self, sort=True):
        if not self.dependencies:
            raise ValueError(
                "No dependencies to install. Add dependencies or remove the"
                " `install_dependencies()` call in the package template."
            )
        pkgs = sorted(self.dependencies) if sort else self.dependencies

        if self.pkg_manager == 'apt':
            return apt_install.render(
                pkgs=pkgs,
                apt_opts=self.__dict__.get('apt_opts'),
                sort=True,
            )
        elif self.pkg_manager == 'yum':
            return yum_install.render(
                pkgs=pkgs,
                yum_opts=self.__dict__.get('yum_opts'),
                sort=True,
            )

    def install_debs(self):
        debs = self._get_debs()
        if not debs:
            raise ValueError(
                "No .deb files to install. Add .deb files or remove the"
                " `install_debs()` call in the package template."
            )
        return deb_install.render(debs=debs)

    def render_run(self):
        return jinja2.Template(self.run).render({self.name: self})

    def render_env(self):
        """Return dictionary with rendered keys and values."""
        return {
            jinja2.Template(k).render({self.name: self}):
            jinja2.Template(v).render({self.name: self})
            for k, v in self.env.items()
        } if self.env else self.env
<commit_msg>removed --no-install-recommends from default, because it was causing trouble with cython<commit_after>""""""

from copy import deepcopy
import os
import posixpath

import jinja2

from neurodocker.utils import load_yaml

GENERIC_VERSION = 'generic'

apt_install = """apt-get update -qq
apt-get install -y {{ apt_opts|default('-q', true) }} \\\
{% for pkg in pkgs %}
    {% if not loop.last -%}
    {{ pkg }} \\\
    {%- else -%}
    {{ pkg }}
    {%- endif -%}
{% endfor %}
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
"""
apt_install = jinja2.Template(apt_install)

yum_install = """yum install -y {{ yum_opts|default('-q', true) }} \\\
{% for pkg in pkgs %}
    {% if not loop.last -%}
    {{ pkg }} \\\
    {%- else -%}
    {{ pkg }}
    {%- endif -%}
{% endfor %}
yum clean packages
rm -rf /var/cache/yum/* /tmp/* /var/tmp/*
"""
yum_install = jinja2.Template(yum_install)

deb_install = """{% for deb_url in debs -%}
curl -sSL --retry 5 -o /tmp/toinstall.deb {{ deb_url }}
dpkg -i /tmp/toinstall.deb
rm /tmp/toinstall.deb
{% endfor -%}
apt-get install -f
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
"""
deb_install = jinja2.Template(deb_install)

def _load_global_specs():

    def load_global_specs(glob_pattern):
        import glob

        def load_interface_spec(filepath):
            _, filename = os.path.split(filepath)
            key, _ = os.path.splitext(filename)
            return key, load_yaml(filepath)

        interface_yamls = glob.glob(glob_pattern)
        instructions = {}
        for ff in interface_yamls:
            key, data = load_interface_spec(ff)
            instructions[key] = data
        return instructions

    base_path = os.path.dirname(os.path.realpath(__file__))
    glob_pattern = os.path.join(base_path, '..', 'templates', '*.yaml')
    return load_global_specs(glob_pattern)

_global_specs = _load_global_specs()

def _interface_exists_in_yaml(name):
    return name in _global_specs.keys()

class _Resolver:
    """
    Parameters
    ----------
    d : dict
    """
    def __init__(self, d):
        self._d = d
        self._generic_only = self._d.keys() == {GENERIC_VERSION}

    @property
    def versions(self):
        return set(self._d.keys())

    def version_exists(self, version):
        if self._generic_only:
            return True
        else:
            return version in self.versions

    def check_version_exists(self, version):
        if not self.version_exists(version):
            raise ValueError("version '{}' not found.".format(version))

    def get_version_key(self, version):
        """Return version key to use given a specific version. For example,
        if a dictionary only has instructions for version 'generic', will
        return 'generic' given any version string.

        Raises
        ------
        `ValueError` if no key could be found for requested version.
        """
        if self._generic_only:
            return GENERIC_VERSION
        else:
            self.check_version_exists(version)
            return version

    def version_has_method(self, version, method):
        version_key = self.get_version_key(version)
        return method in self._d[version_key].keys()

    def check_version_has_method(self, version, method):
        if not self.version_has_method(version, method):
            raise ValueError(
                "version '{}' does not have method '{}'"
                .format(version, method)
            )

    def version_method_has_instructions(self, version, method):
        version_key = self.get_version_key(version)
        self.check_version_has_method(version_key, method)
        return 'instructions' in self._d[version_key][method].keys()

    def check_version_method_has_instructions(self, version, method):
        if not self.version_method_has_instructions(version, method):
            raise ValueError(
                "installation method '{}' for version '{}' does not have an"
                " 'instructions' key.".format(method, version)
            )

    def binaries_has_url(self, version):
        version_key = self.get_version_key(version)
        if self.version_has_method(version_key, 'binaries'):
            try:
                urls = self._d[version_key]['binaries']['urls'].keys()
                return version in urls
            except KeyError:
                raise ValueError(
                    "no binary URLs defined for version '{}'".format(version)
                )
        else:
            raise ValueError(
                "no binary installation method defined for version '{}"
                .format(version)
            )

    def check_binaries_has_url(self, version):
        if not self.binaries_has_url(version):
            version_key = self.get_version_key(version)
            valid_vers = self._d[version_key]['binaries']['urls']
            raise ValueError(
                "URL not found for version '{}'. Valid versions are {}"
                .format(version, ', '.join(valid_vers))
            )

    def binaries_url(self, version):
        self.check_binaries_has_url(version)
        version_key = self.get_version_key(version)
        return self._d[version_key]['binaries']['urls'][version]

class _BaseInterface:
    """Base class for interface objects."""

    def __init__(self, name, version, pkg_manager, method='binaries',
                 install_path=None, **kwargs):
        self._name = name
        self._version = version
        self._pkg_manager = pkg_manager
        self._method = method
        self._install_path = install_path
        self.__dict__.update(**kwargs)

        if not _interface_exists_in_yaml(self._name):
            raise ValueError(
                "No YAML entry for package '{}'".format(self._name)
            )
        self._resolver = _Resolver(_global_specs[self._name])

        self._version_key = self._resolver.get_version_key(self._version)
        self._resolver.check_version_exists(self._version)
        self._resolver.check_version_has_method(self._version, self._method)
        self._resolver.check_version_method_has_instructions(
            self._version, self._method
        )

        if method == 'binaries':
            self.binaries_url = self._resolver.binaries_url(self._version)

        self._instance_specs = deepcopy(
            _global_specs[self._name][self._version_key][self._method]
        )

        self._run = self._instance_specs['instructions']
        self._dependencies = self._get_dependencies()

        self._env = self._instance_specs.get('env', None)

        # Set default curl options for all interfaces.
        self.__dict__.setdefault("curl_opts", "-fsSL --retry 5")

    @property
    def _pretty_name(self):
        return self.__class__.__name__

    @property
    def name(self):
        return self._name

    @property
    def version(self):
        return self._version

    @property
    def versions(self):
        return self._resolver.versions

    @property
    def pkg_manager(self):
        return self._pkg_manager

    @property
    def method(self):
        return self._method

    @property
    def env(self):
        return self._env

    @property
    def run(self):
        return self._run

    @property
    def install_path(self):
        if self._install_path is None:
            path = posixpath.join(posixpath.sep, 'opt', '{}-{}')
            return path.format(self._name, self._version)
        return self._install_path

    @property
    def dependencies(self):
        return self._dependencies

    def _get_dependencies(self):
        if 'dependencies' not in self._instance_specs.keys():
            return None
        if self._instance_specs['dependencies'] is None:
            return None
        try:
            deps = self._instance_specs['dependencies'][self._pkg_manager]
            return deps.split() if deps else None
        except KeyError:
            return None

    def _get_debs(self):
        if 'dependencies' not in self._instance_specs.keys():
            return None
        if self._instance_specs['dependencies'] is None:
            return None
        try:
            debs = self._instance_specs['dependencies']['debs']
            return debs if debs else None
        except KeyError:
            return None

    def install_dependencies(self, sort=True):
        if not self.dependencies:
            raise ValueError(
                "No dependencies to install. Add dependencies or remove the"
                " `install_dependencies()` call in the package template."
            )
        pkgs = sorted(self.dependencies) if sort else self.dependencies

        if self.pkg_manager == 'apt':
            return apt_install.render(
                pkgs=pkgs,
                apt_opts=self.__dict__.get('apt_opts'),
                sort=True,
            )
        elif self.pkg_manager == 'yum':
            return yum_install.render(
                pkgs=pkgs,
                yum_opts=self.__dict__.get('yum_opts'),
                sort=True,
            )

    def install_debs(self):
        debs = self._get_debs()
        if not debs:
            raise ValueError(
                "No .deb files to install. Add .deb files or remove the"
                " `install_debs()` call in the package template."
            )
        return deb_install.render(debs=debs)

    def render_run(self):
        return jinja2.Template(self.run).render({self.name: self})

    def render_env(self):
        """Return dictionary with rendered keys and values."""
        return {
            jinja2.Template(k).render({self.name: self}):
            jinja2.Template(v).render({self.name: self})
            for k, v in self.env.items()
        } if self.env else self.env
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" Classification and decoding related tools """

import numpy as np
from functools import reduce
from sklearn.feature_selection.univariate_selection import SelectKBest
import re

def feature_selection(feat_select, X, y):
    """" Implements various kinds of feature selection """
    # K-best
    if re.match('.*-best', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        selector = SelectKBest(k=n)

        import warnings
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=UserWarning)
            features_selected = np.where(
                selector.fit(X, y).get_support() == True)[0]

    elif re.match('.*-randombest', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        from random import shuffle
        features = range(0, X.shape[1])
        shuffle(features)

        features_selected = features[:n]

    return features_selected

def get_score(X, y, clf, scoring = 'accuracy'):
    from sklearn.preprocessing import binarize
    from sklearn.metrics import accuracy_score

    if scoring == 'accuracy':
        score = accuracy_score(y, binarize(clf.predict(X), 0.5))
    else:
        score = clf.score(X, y)

    return score

def classify_by_features(dataset, features, studies=None, method='SVM',
                         scikit_classifier=None):
    pass

def regularize(X, method='scale'):
    if method == 'scale':
        from sklearn import preprocessing
        return preprocessing.scale(X, with_mean=False)
    else:
        raise Exception('Unrecognized regularization method')

def get_studies_by_regions(dataset, masks, threshold=0.08,
    remove_overlap=True, studies=None,
    features=None, regularization="scale"):
    """ Set up data for a classification task given a set of masks

        Given a set of masks, this function retrieves studies associated with each
        mask at the specified threshold, optionally removes overlap and filters by studies
        and features, and returns studies by feature matrix (X) and class labels (y)

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            threshold: percentage of voxels active within the mask for study to be included
            remove_overlap: A boolean indicating if studies studies that appear in more than
                one mask should be excluded
            studies: An optional list of study names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            features: An optional list of feature names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            regularize: Optional boolean indicating if X should be regularized

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    import nibabel as nib
    import os

    # Load masks using NiBabel

    try:
        loaded_masks = [nib.load(os.path.relpath(m)) for m in masks]
    except OSError:
        print 'Error loading masks. Check the path'

    # Get a list of studies that activate for each mask file--i.e.,  a list of
    # lists

    grouped_ids = [dataset.get_ids_by_mask(m, threshold=threshold)
                   for m in loaded_masks]

    # Flattened ids

    flat_ids = reduce(lambda a, b: a + b, grouped_ids)

    # Remove duplicates

    if remove_overlap:
        import collections
        flat_ids = [id for (id, count) in
                    collections.Counter(flat_ids).items() if count == 1]
        grouped_ids = [[x for x in m if x in flat_ids] for m in
                       grouped_ids]  # Remove

    # Create class label(y)
    y = [[idx] * len(ids) for (idx, ids) in enumerate(grouped_ids)]
    y = reduce(lambda a, b: a + b, y)  # Flatten
    y = np.array(y)

    # Extract feature set for each class separately
    X = [dataset.get_feature_data(ids=group_ids, features=None) for group_ids in grouped_ids]

    X = np.vstack(tuple(X))

    if regularization:
        X = regularize(X, method=regularization)

    return (X, y)

def get_feature_order(dataset, features):
    """ Returns a list with the order that features requested appear in dataset """
    all_features = dataset.get_feature_names()

    i = [all_features.index(f) for f in features]

    return i

def classify_regions(dataset, masks, method='ERF', threshold=0.08,
    remove_overlap=True, regularization='scale',
    output='summary', studies=None, features=None,
    class_weight='auto', classifier=None,
    cross_val='4-Fold', param_grid=None, scoring='accuracy'):
    """ Perform classification on specified regions

        Given a set of masks, this function retrieves studies associated with each
        mask at the specified threshold, optionally removes overlap and filters by studies
        and features. Then it trains an algorithm to classify studies based on features
        and tests performance.

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            method: a string indicating which method to used.
                'SVM': Support Vector Classifier with rbf kernel
                'ERF': Extremely Randomized Forest classifier
                'Dummy': A dummy classifier using stratified classes as predictor
            threshold: percentage of voxels active within the mask for study to be included
            remove_overlap: A boolean indicating if studies studies that appear in more than
                one mask should be excluded
            regularization: A string indicating type of regularization to use. If None
                performs no regularization.
                'scale': Unit scale without demeaning
            output: A string indicating output type
                'summary': Dictionary with summary statistics including score and n
                'summary_clf': Same as above but also includes classifier
                'clf': Only returns classifier
                Warning: using cv without grid will return an untrained classifier
            studies: An optional list of study names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            features: An optional list of feature names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            class_weight: Parameter to pass to classifier determining how to weight classes
            classifier: An optional sci-kit learn classifier to use instead of pre-set up
                classifiers set up using 'method'
            cross_val: A string indicating type of cross validation to use. Can also pass a
                scikit_classifier
            param_grid: A dictionary indicating which parameters to optimize using GridSearchCV
                If None, no GridSearch will be used

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    (X, y) = get_studies_by_regions(dataset, masks, threshold,
                                    remove_overlap, studies, features, regularization=regularization)

    return classify(X, y, method, classifier, output, cross_val,
                    class_weight, scoring=scoring, param_grid=param_grid)

def classify(X, y, clf_method='ERF', classifier=None, output='summary',
             cross_val=None, class_weight=None, regularization=None,
             param_grid=None, scoring='accuracy', refit_all=True, feat_select=None):
    """ Wrapper for scikit-learn classification functions
    Imlements various types of classification and cross validation """

    # Build classifier
    clf = Classifier(clf_method, classifier, param_grid)

    # Fit & test model with or without cross-validation
    if cross_val is not None:
        score = clf.cross_val_fit(X, y, cross_val, scoring=scoring, feat_select=feat_select, class_weight=class_weight)
    else:
        # Does not support scoring function
        score = clf.fit(X, y, class_weight=class_weight).score(X, y)

    # Return some stuff...
    from collections import Counter

    if output == 'clf':
        return clf
    else:
        if output == 'summary':
            output = {'score': score, 'n': dict(Counter(y))}
        elif output == 'summary_clf':
            output = {'score': score, 'n': dict(Counter(y)), 'clf': clf, 'features_selected': clf.features_selected}

        return output

class Classifier:

    def __init__(self, clf_method='ERF', classifier=None, param_grid=None):
        """ Initialize a new classifier instance """

        # Set classifier
        self.features_selected = None

        if classifier is not None:
            self.clf = classifier

            from sklearn.svm import LinearSVC
            import random
            if isinstance(self.clf, LinearSVC):
                self.clf.set_params().random_state = random.randint(0, 200)
        else:
            if clf_method == 'SVM':
                from sklearn import svm
                self.clf = svm.SVC()
            elif clf_method == 'ERF':
                from sklearn.ensemble import ExtraTreesClassifier
                self.clf = ExtraTreesClassifier(n_estimators=100,
                                                max_depth=None, min_samples_split=1,
                                                random_state=0)
            elif clf_method == 'GBC':
                from sklearn.ensemble import GradientBoostingClassifier
                self.clf = GradientBoostingClassifier(n_estimators=100,
                                                      max_depth=1)
            elif clf_method == 'Dummy':
                from sklearn.dummy import DummyClassifier
                self.clf = DummyClassifier(strategy='stratified')
            else:
                raise Exception('Unrecognized classification method')

        if isinstance(param_grid, dict):
            from sklearn.grid_search import GridSearchCV
            self.clf = GridSearchCV(estimator=self.clf,
                                    param_grid=param_grid)

    def fit(self, X, y, cv=None, class_weight = 'auto'):
        """ Fits X to outcomes y, using clf """

        # Incorporate error checking such as :
        # if isinstance(self.classifier, ScikitClassifier):
        #     do one thingNone
        # otherwiseNone.

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y = y)

        self.clf = self.clf.fit(X, y)

        return self.clf

    def set_class_weight(self, class_weight='auto', y=None):
        """ Sets the class_weight of the classifier to match X """

        if class_weight == None:
            cw = None

            try:
                self.clf.set_params(class_weight = cw)
            except ValueError:
                pass

        elif class_weight == 'auto':
            c = np.bincount(y)
            ii = np.nonzero(c)[0]
            c = c / float(c.sum())
            cw = dict(zip(ii[::-1],c[ii]))

            try:
                self.clf.set_params(class_weight = cw)
            except ValueError:
                import warnings
                warnings.warn("Tried to set class_weight, but failed. The classifier probably doesn't support it")

    def cross_val_fit(self, X, y, cross_val='4-Fold', scoring='accuracy', feat_select=None, class_weight = 'auto'):
        """ Fits X to outcomes y, using clf and cv_method """

        from sklearn import cross_validation

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y = y)

        # Set cross validator
        if isinstance(cross_val, basestring):
            if cross_val == '4-Fold':
                self.cver = cross_validation.StratifiedKFold(self.y, 4,
                                                             indices=True)

            elif cross_val == '3-Fold':
                self.cver = cross_validation.StratifiedKFold(self.y, 3,
                                                             indices=True)
            else:
                raise Exception('Unrecognized cross validation method')
        else:
            self.cver = cross_val

        if feat_select is not None:
            self.features_selected = []

        # Perform cross-validated classification
        from sklearn.grid_search import GridSearchCV
        if isinstance(self.clf, GridSearchCV):
            import warnings

            if feat_select is not None:
                warnings.warn(
                    "Cross-validated feature selection not supported witih GridSearchCV")
            self.clf.set_params(cv=self.cver, scoring=scoring)

            with warnings.catch_warnings():
                warnings.simplefilter('ignore', category=UserWarning)
                self.clf = self.clf.fit(X, y)

            self.cvs = self.clf.best_score_
        else:
            self.cvs = self.feat_select_cvs(feat_select=feat_select, scoring=scoring)

        if feat_select is not None:
            fs = feature_selection(
                    feat_select, X, y)
            self.features_selected.append(fs)

            X = X[:, fs]

        self.clf.fit(X, y)

        return self.cvs.mean()

    def feat_select_cvs(self, scoring = None, feat_select=None):
        """ Returns cross validated scores (just like cross_val_score),
        but includes feature selection as part of the cross validation loop """

        scores = []

        for train, test in self.cver:
            X_train, X_test, y_train, y_test = self.X[
                train], self.X[test], self.y[train], self.y[test]

            if feat_select is not None:
                # Get which features are kept
                fs = feature_selection(
                    feat_select, X_train, y_train)

                self.features_selected.append(fs)

                # Filter X to only keep selected features
                X_train, X_test = X_train[
                    :, fs], X_test[:, fs]

            # Set scoring (not implement as accuracy is default)

            # Train classifier
            self.clf.fit(X_train, y_train)

            # Test classifier
            s = get_score(X_test, y_test, self.clf, scoring = scoring)

            scores.append(s)

        return np.array(scores)

    def fit_dataset(self, dataset, y, features=None,
                    feature_type='features'):
        """ Given a dataset, fits either features or voxels to y """

        # Get data from dataset

        if feature_type == 'features':
            X = np.rot90(dataset.feature_table.data.toarray())
        elif feature_type == 'voxels':
            X = np.rot90(dataset.image_table.data.toarray())

        self.sk_classifier.fit(X, y)
<commit_msg>Feature subset fix<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" Classification and decoding related tools """

import numpy as np
from functools import reduce
from sklearn.feature_selection.univariate_selection import SelectKBest
import re

def feature_selection(feat_select, X, y):
    """" Implements various kinds of feature selection """
    # K-best
    if re.match('.*-best', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        selector = SelectKBest(k=n)

        import warnings
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', category=UserWarning)
            features_selected = np.where(
                selector.fit(X, y).get_support() == True)[0]

    elif re.match('.*-randombest', feat_select) is not None:
        n = int(feat_select.split('-')[0])

        from random import shuffle
        features = range(0, X.shape[1])
        shuffle(features)

        features_selected = features[:n]

    return features_selected

def get_score(X, y, clf, scoring = 'accuracy'):
    from sklearn.preprocessing import binarize
    from sklearn.metrics import accuracy_score

    if scoring == 'accuracy':
        score = accuracy_score(y, binarize(clf.predict(X), 0.5))
    else:
        score = clf.score(X, y)

    return score

def classify_by_features(dataset, features, studies=None, method='SVM',
                         scikit_classifier=None):
    pass

def regularize(X, method='scale'):
    if method == 'scale':
        from sklearn import preprocessing
        return preprocessing.scale(X, with_mean=False)
    else:
        raise Exception('Unrecognized regularization method')

def get_studies_by_regions(dataset, masks, threshold=0.08,
    remove_overlap=True, studies=None,
    features=None, regularization="scale"):
    """ Set up data for a classification task given a set of masks

        Given a set of masks, this function retrieves studies associated with each
        mask at the specified threshold, optionally removes overlap and filters by studies
        and features, and returns studies by feature matrix (X) and class labels (y)

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            threshold: percentage of voxels active within the mask for study to be included
            remove_overlap: A boolean indicating if studies studies that appear in more than
                one mask should be excluded
            studies: An optional list of study names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            features: An optional list of feature names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            regularize: Optional boolean indicating if X should be regularized

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    import nibabel as nib
    import os

    # Load masks using NiBabel

    try:
        loaded_masks = [nib.load(os.path.relpath(m)) for m in masks]
    except OSError:
        print 'Error loading masks. Check the path'

    # Get a list of studies that activate for each mask file--i.e.,  a list of
    # lists

    grouped_ids = [dataset.get_ids_by_mask(m, threshold=threshold)
                   for m in loaded_masks]

    # Flattened ids

    flat_ids = reduce(lambda a, b: a + b, grouped_ids)

    # Remove duplicates

    if remove_overlap:
        import collections
        flat_ids = [id for (id, count) in
                    collections.Counter(flat_ids).items() if count == 1]
        grouped_ids = [[x for x in m if x in flat_ids] for m in
                       grouped_ids]  # Remove

    # Create class label(y)
    y = [[idx] * len(ids) for (idx, ids) in enumerate(grouped_ids)]
    y = reduce(lambda a, b: a + b, y)  # Flatten
    y = np.array(y)

    # Extract feature set for each class separately
    X = [dataset.get_feature_data(ids=group_ids, features=features) for group_ids in grouped_ids]

    X = np.vstack(tuple(X))

    if regularization:
        X = regularize(X, method=regularization)

    return (X, y)

def get_feature_order(dataset, features):
    """ Returns a list with the order that features requested appear in dataset """
    all_features = dataset.get_feature_names()

    i = [all_features.index(f) for f in features]

    return i

def classify_regions(dataset, masks, method='ERF', threshold=0.08,
    remove_overlap=True, regularization='scale',
    output='summary', studies=None, features=None,
    class_weight='auto', classifier=None,
    cross_val='4-Fold', param_grid=None, scoring='accuracy'):
    """ Perform classification on specified regions

        Given a set of masks, this function retrieves studies associated with each
        mask at the specified threshold, optionally removes overlap and filters by studies
        and features. Then it trains an algorithm to classify studies based on features
        and tests performance.

        Args:
            dataset: a Neurosynth dataset
            maks: a list of paths to Nifti masks
            method: a string indicating which method to used.
                'SVM': Support Vector Classifier with rbf kernel
                'ERF': Extremely Randomized Forest classifier
                'Dummy': A dummy classifier using stratified classes as predictor
            threshold: percentage of voxels active within the mask for study to be included
            remove_overlap: A boolean indicating if studies studies that appear in more than
                one mask should be excluded
            regularization: A string indicating type of regularization to use. If None
                performs no regularization.
                'scale': Unit scale without demeaning
            output: A string indicating output type
                'summary': Dictionary with summary statistics including score and n
                'summary_clf': Same as above but also includes classifier
                'clf': Only returns classifier
                Warning: using cv without grid will return an untrained classifier
            studies: An optional list of study names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            features: An optional list of feature names used to constrain the set used in
                classification. If None, will use all features in the dataset.
            class_weight: Parameter to pass to classifier determining how to weight classes
            classifier: An optional sci-kit learn classifier to use instead of pre-set up
                classifiers set up using 'method'
            cross_val: A string indicating type of cross validation to use. Can also pass a
                scikit_classifier
            param_grid: A dictionary indicating which parameters to optimize using GridSearchCV
                If None, no GridSearch will be used

        Returns:
            A tuple (X, y) of np arrays.
            X is a feature by studies matrix and y is a vector of class labels
    """

    (X, y) = get_studies_by_regions(dataset, masks, threshold,
                                    remove_overlap, studies, features, regularization=regularization)

    return classify(X, y, method, classifier, output, cross_val,
                    class_weight, scoring=scoring, param_grid=param_grid)

def classify(X, y, clf_method='ERF', classifier=None, output='summary',
             cross_val=None, class_weight=None, regularization=None,
             param_grid=None, scoring='accuracy', refit_all=True, feat_select=None):
    """ Wrapper for scikit-learn classification functions
    Imlements various types of classification and cross validation """

    # Build classifier
    clf = Classifier(clf_method, classifier, param_grid)

    # Fit & test model with or without cross-validation
    if cross_val is not None:
        score = clf.cross_val_fit(X, y, cross_val, scoring=scoring, feat_select=feat_select, class_weight=class_weight)
    else:
        # Does not support scoring function
        score = clf.fit(X, y, class_weight=class_weight).score(X, y)

    # Return some stuff...
    from collections import Counter

    if output == 'clf':
        return clf
    else:
        if output == 'summary':
            output = {'score': score, 'n': dict(Counter(y))}
        elif output == 'summary_clf':
            output = {'score': score, 'n': dict(Counter(y)), 'clf': clf, 'features_selected': clf.features_selected}

        return output

class Classifier:

    def __init__(self, clf_method='ERF', classifier=None, param_grid=None):
        """ Initialize a new classifier instance """

        # Set classifier
        self.features_selected = None

        if classifier is not None:
            self.clf = classifier

            from sklearn.svm import LinearSVC
            import random
            if isinstance(self.clf, LinearSVC):
                self.clf.set_params().random_state = random.randint(0, 200)
        else:
            if clf_method == 'SVM':
                from sklearn import svm
                self.clf = svm.SVC()
            elif clf_method == 'ERF':
                from sklearn.ensemble import ExtraTreesClassifier
                self.clf = ExtraTreesClassifier(n_estimators=100,
                                                max_depth=None, min_samples_split=1,
                                                random_state=0)
            elif clf_method == 'GBC':
                from sklearn.ensemble import GradientBoostingClassifier
                self.clf = GradientBoostingClassifier(n_estimators=100,
                                                      max_depth=1)
            elif clf_method == 'Dummy':
                from sklearn.dummy import DummyClassifier
                self.clf = DummyClassifier(strategy='stratified')
            else:
                raise Exception('Unrecognized classification method')

        if isinstance(param_grid, dict):
            from sklearn.grid_search import GridSearchCV
            self.clf = GridSearchCV(estimator=self.clf,
                                    param_grid=param_grid)

    def fit(self, X, y, cv=None, class_weight = 'auto'):
        """ Fits X to outcomes y, using clf """

        # Incorporate error checking such as :
        # if isinstance(self.classifier, ScikitClassifier):
        #     do one thingNone
        # otherwiseNone.

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y = y)

        self.clf = self.clf.fit(X, y)

        return self.clf

    def set_class_weight(self, class_weight='auto', y=None):
        """ Sets the class_weight of the classifier to match X """

        if class_weight == None:
            cw = None

            try:
                self.clf.set_params(class_weight = cw)
            except ValueError:
                pass

        elif class_weight == 'auto':
            c = np.bincount(y)
            ii = np.nonzero(c)[0]
            c = c / float(c.sum())
            cw = dict(zip(ii[::-1],c[ii]))

            try:
                self.clf.set_params(class_weight = cw)
            except ValueError:
                import warnings
                warnings.warn("Tried to set class_weight, but failed. The classifier probably doesn't support it")

    def cross_val_fit(self, X, y, cross_val='4-Fold', scoring='accuracy', feat_select=None, class_weight = 'auto'):
        """ Fits X to outcomes y, using clf and cv_method """

        from sklearn import cross_validation

        self.X = X
        self.y = y

        self.set_class_weight(class_weight=class_weight, y = y)

        # Set cross validator
        if isinstance(cross_val, basestring):
            if cross_val == '4-Fold':
                self.cver = cross_validation.StratifiedKFold(self.y, 4,
                                                             indices=True)

            elif cross_val == '3-Fold':
                self.cver = cross_validation.StratifiedKFold(self.y, 3,
                                                             indices=True)
            else:
                raise Exception('Unrecognized cross validation method')
        else:
            self.cver = cross_val

        if feat_select is not None:
            self.features_selected = []

        # Perform cross-validated classification
        from sklearn.grid_search import GridSearchCV
        if isinstance(self.clf, GridSearchCV):
            import warnings

            if feat_select is not None:
                warnings.warn(
                    "Cross-validated feature selection not supported witih GridSearchCV")
            self.clf.set_params(cv=self.cver, scoring=scoring)

            with warnings.catch_warnings():
                warnings.simplefilter('ignore', category=UserWarning)
                self.clf = self.clf.fit(X, y)

            self.cvs = self.clf.best_score_
        else:
            self.cvs = self.feat_select_cvs(feat_select=feat_select, scoring=scoring)

        if feat_select is not None:
            fs = feature_selection(
                    feat_select, X, y)
            self.features_selected.append(fs)

            X = X[:, fs]

        self.clf.fit(X, y)

        return self.cvs.mean()

    def feat_select_cvs(self, scoring = None, feat_select=None):
        """ Returns cross validated scores (just like cross_val_score),
        but includes feature selection as part of the cross validation loop """

        scores = []

        for train, test in self.cver:
            X_train, X_test, y_train, y_test = self.X[
                train], self.X[test], self.y[train], self.y[test]

            if feat_select is not None:
                # Get which features are kept
                fs = feature_selection(
                    feat_select, X_train, y_train)

                self.features_selected.append(fs)

                # Filter X to only keep selected features
                X_train, X_test = X_train[
                    :, fs], X_test[:, fs]

            # Set scoring (not implement as accuracy is default)

            # Train classifier
            self.clf.fit(X_train, y_train)

            # Test classifier
            s = get_score(X_test, y_test, self.clf, scoring = scoring)

            scores.append(s)

        return np.array(scores)

    def fit_dataset(self, dataset, y, features=None,
                    feature_type='features'):
        """ Given a dataset, fits either features or voxels to y """

        # Get data from dataset

        if feature_type == 'features':
            X = np.rot90(dataset.feature_table.data.toarray())
        elif feature_type == 'voxels':
            X = np.rot90(dataset.image_table.data.toarray())

        self.sk_classifier.fit(X, y)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from .celery_broker import app
import celery.signals
import os
import sys
import time
import json
import traceback
import numpy
import hashlib

import next.utils
import next.constants
from next.constants import DEBUG_ON

from next.database_client.DatabaseAPI import DatabaseAPI, DatabaseException
from next.logging_client.LoggerAPI import LoggerAPI

from next.apps.Butler import Butler
import next.lib.pijemont.verifier as verifier

db, ell = None, None

def worker_connect_db():
    global db, ell
    print("Initializing worker database connection")
    backoff_dt = 0.01
    while True:
        try:
            db, ell = DatabaseAPI(), LoggerAPI()
            break
        except DatabaseException as e:
            print("Failed to connect to database ({}), retrying in {}s".format(e, backoff_dt))
            time.sleep(backoff_dt)
            backoff_dt *= 2

# if we're not using celery, just initialize the database globally
if next.constants.CELERY_ON:
    worker_connect_db()

# runs for each worker process spawned by Celery
# we initialize the DatabaseAPI per worker here
@celery.signals.worker_process_init.connect
def on_connect(**kwargs):
    # make sure every worker has a different random seed (very important for randomized algorithms)
    numpy.random.seed()

    worker_connect_db()

# runs when each Celery worker process shuts down
# we'll close the database connections here
@celery.signals.worker_process_shutdown.connect
def on_shutdown(**kwargs):
    global db, ell
    if db:
        print("Closing worker's database connections")
        db.close()
    if ell:
        print("Closing worker's logger connections")
        ell.close()

class App_Wrapper:
        def __init__(self, app_id, exp_uid, db, ell):
                self.app_id = app_id
                self.exp_uid = exp_uid
                self.next_app = next.utils.get_app(app_id, exp_uid, db, ell)
                self.butler = Butler(app_id, exp_uid, self.next_app.myApp.TargetManager, db, ell)

        def getModel(self, args_in_json):
                response, dt = next.utils.timeit(self.next_app.getModel)(self.next_app.exp_uid, args_in_json)
                args_out_json,didSucceed,message = response
                args_out_dict = json.loads(args_out_json)
                meta = args_out_dict.get('meta',{})
                if 'log_entry_durations' in meta.keys():
                        self.log_entry_durations = meta['log_entry_durations']
                        self.log_entry_durations['timestamp'] = next.utils.datetimeNow()
                return args_out_dict['args']

# Main application task
def apply(app_id, exp_uid, task_name, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	# modify args_in
	if task_name == 'processAnswer':
		args_in_dict = json.loads(args_in_json)
		args_in_dict['args']['timestamp_answer_received'] = enqueue_timestamp
		args_in_json = json.dumps(args_in_dict)
	# get stateless app
	next_app = next.utils.get_app(app_id, exp_uid, db, ell)
	# pass it to a method
	method = getattr(next_app, task_name)
	response, dt = next.utils.timeit(method)(exp_uid, args_in_json)
        args_out_json,didSucceed,message = response
        args_out_dict = json.loads(args_out_json)
	if 'args' in args_out_dict:
		return_value = (json.dumps(args_out_dict['args']),didSucceed,message)
		meta = args_out_dict.get('meta',{})
		if 'log_entry_durations' in meta:
			log_entry_durations = meta['log_entry_durations']
			log_entry_durations['app_duration'] = dt
			log_entry_durations['duration_enqueued'] = time_enqueued
			log_entry_durations['timestamp'] = next.utils.datetimeNow()
			ell.log( app_id+':ALG-DURATION', log_entry_durations  )
	else:
		return_value = (args_out_json,didSucceed,message)
	print '#### Finished %s,  time_enqueued=%s,  execution_time=%s ####' % (task_name,time_enqueued,dt)
	return return_value

def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.
        dir, _ = os.path.split(__file__)
        reference_dict,errs = verifier.load_doc('{}/myApp.yaml'.format(app_id, app_id),"apps/")
        if len(errs) > 0:
                raise Exception("App YAML format errors: \n{}".format(str(errs)))
        args_dict = verifier.verify(args_in_json, reference_dict['getStats']['args'])
        stat_id = args_dict['args'].get('stat_id','none')

        stat_args = args_dict['args']

        hash_object = hashlib.md5(stat_id+'_'+json.dumps(stat_args['params']))
        stat_uid = hash_object.hexdigest()
        stat_uid += '_' + exp_uid

        app = App_Wrapper(app_id, exp_uid, db, ell)
        cached_doc = app.butler.dashboard.get(uid=stat_uid)
        cached_response = None
        if (int(stat_args.get('force_recompute',0))==0) and (cached_doc is not None):
          delta_datetime = (next.utils.datetimeNow() - next.utils.str2datetime(cached_doc['timestamp']))
          if delta_datetime.seconds < next.constants.DASHBOARD_STALENESS_IN_SECONDS:
            cached_response = json.loads(cached_doc['data_dict'])
            if 'meta' not in cached_response:
              cached_response['meta']={}
            cached_response['meta']['cached'] = 1
            if delta_datetime.seconds/60<1:
                cached_response['meta']['last_dashboard_update'] = '<1 minute ago'
            else:
                cached_response['meta']['last_dashboard_update'] = str(delta_datetime.seconds/60)+' minutes ago'

        if cached_response==None:
            dashboard_string = 'apps.' + app_id + '.dashboard.Dashboard'
            dashboard_module = __import__(dashboard_string, fromlist=[''])
            dashboard = getattr(dashboard_module, 'MyAppDashboard')
            dashboard = dashboard(db, ell)
            stats_method = getattr(dashboard, stat_id)
            response,dt = next.utils.timeit(stats_method)(app,app.butler,**args_dict['args']['params'])

            save_dict = {'exp_uid':app.exp_uid,
                    'stat_uid':stat_uid,
                    'timestamp':next.utils.datetime2str(next.utils.datetimeNow()),
                    'data_dict':json.dumps(response)}
            app.butler.dashboard.set_many(uid=stat_uid,key_value_dict=save_dict)

            # update the admin timing with the timing of a getModel
            if hasattr(app, 'log_entry_durations'):
                app.log_entry_durations['app_duration'] = dt
                app.log_entry_durations['duration_enqueued'] = time_enqueued
                app.butler.ell.log(app.app_id+':ALG-DURATION', app.log_entry_durations)
        else:
            response = cached_response

        if DEBUG_ON:
            next.utils.debug_print('#### Finished Dashboard %s, time_enqueued=%s,  execution_time=%s ####' % (stat_id, time_enqueued, dt), color='white')
	return json.dumps(response), True, ''

def apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name, args, namespace, job_uid, enqueue_timestamp, time_limit):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	try:
		print '>>>>>>>> Starting namespace:%s,  job_uid=%s,  time_enqueued=%s <<<<<<<<<' % (namespace,job_uid,time_enqueued)
                # get stateless app
                next_app = next.utils.get_app(app_id, exp_uid, db, ell)
                target_manager = next_app.myApp.TargetManager
                next_alg = next.utils.get_app_alg(app_id, alg_id)
                butler = Butler(app_id, exp_uid, target_manager, db, ell, alg_label, alg_id)
		response,dt = next.utils.timeit(getattr(next_alg, task_name))(butler, args)
                log_entry_durations = { 'exp_uid':exp_uid,'alg_label':alg_label,'task':'daemonProcess','duration':dt }
                log_entry_durations.update(butler.algorithms.getDurations())
                log_entry_durations['app_duration'] = dt
                log_entry_durations['duration_enqueued'] = time_enqueued
                log_entry_durations['timestamp'] = next.utils.datetimeNow()
                ell.log( app_id+':ALG-DURATION', log_entry_durations)
		print '########## Finished namespace:%s,  job_uid=%s,  time_enqueued=%s,  execution_time=%s ##########' % (namespace,job_uid,time_enqueued,dt)
		return
	except Exception, error:
		exc_type, exc_value, exc_traceback = sys.exc_info()
                print "tasks Exception: {} {}".format(error, traceback.format_exc())
                traceback.print_tb(exc_traceback)

		# error = traceback.format_exc()
		# log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':next.utils.datetimeNow() }
		# ell.log( app_id+':APP-EXCEPTION', log_entry  )
		return None

# If celery isn't off, celery-wrap the functions so they can be called with apply_async
if next.constants.CELERY_ON:
        apply = app.task(apply)
        apply_dashboard = app.task(apply_dashboard)
        apply_sync_by_namespace = app.task(apply_sync_by_namespace)

<commit_msg>tasks.py: Implement @stsievert's suggestions<commit_after>from __future__ import absolute_import
from .celery_broker import app
import celery.signals
import os
import sys
import time
import json
import traceback
import numpy
import hashlib

import next.utils
import next.constants
from next.constants import DEBUG_ON

from next.database_client.DatabaseAPI import DatabaseAPI, DatabaseException
from next.logging_client.LoggerAPI import LoggerAPI

from next.apps.Butler import Butler
import next.lib.pijemont.verifier as verifier

db, ell = None, None

def worker_connect_db():
    next.utils.debug_print("Initializing worker database connection")
    backoff_dt = 0.01
    while True:
        try:
            db, ell = DatabaseAPI(), LoggerAPI()
            break
        except DatabaseException as e:
            next.utils.debug_print("Failed to connect to database ({}), retrying in {}s".format(e, backoff_dt))
            time.sleep(backoff_dt)
            backoff_dt *= 2

            # If we've tried too many times, make the database failure /loud/.
            if backoff_dt > 1:
                raise e

    return db, ell

# if we're not using celery, just initialize the database globally
if next.constants.CELERY_ON:
    db, ell = worker_connect_db()

# runs for each worker process spawned by Celery
# we initialize the DatabaseAPI per worker here
@celery.signals.worker_process_init.connect
def on_connect(**kwargs):
    global db, ell
    # make sure every worker has a different random seed (very important for randomized algorithms)
    numpy.random.seed()

    db, ell = worker_connect_db()

# runs when each Celery worker process shuts down
# we'll close the database connections here
@celery.signals.worker_process_shutdown.connect
def on_shutdown(**kwargs):
    global db, ell
    if db:
        next.utils.debug_print("Closing worker's database connections")
        db.close()
    if ell:
        next.utils.debug_print("Closing worker's logger connections")
        ell.close()

class App_Wrapper:
        def __init__(self, app_id, exp_uid, db, ell):
                self.app_id = app_id
                self.exp_uid = exp_uid
                self.next_app = next.utils.get_app(app_id, exp_uid, db, ell)
                self.butler = Butler(app_id, exp_uid, self.next_app.myApp.TargetManager, db, ell)

        def getModel(self, args_in_json):
                response, dt = next.utils.timeit(self.next_app.getModel)(self.next_app.exp_uid, args_in_json)
                args_out_json,didSucceed,message = response
                args_out_dict = json.loads(args_out_json)
                meta = args_out_dict.get('meta',{})
                if 'log_entry_durations' in meta.keys():
                        self.log_entry_durations = meta['log_entry_durations']
                        self.log_entry_durations['timestamp'] = next.utils.datetimeNow()
                return args_out_dict['args']

# Main application task
def apply(app_id, exp_uid, task_name, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	# modify args_in
	if task_name == 'processAnswer':
		args_in_dict = json.loads(args_in_json)
		args_in_dict['args']['timestamp_answer_received'] = enqueue_timestamp
		args_in_json = json.dumps(args_in_dict)
	# get stateless app
	next_app = next.utils.get_app(app_id, exp_uid, db, ell)
	# pass it to a method
	method = getattr(next_app, task_name)
	response, dt = next.utils.timeit(method)(exp_uid, args_in_json)
        args_out_json,didSucceed,message = response
        args_out_dict = json.loads(args_out_json)
	if 'args' in args_out_dict:
		return_value = (json.dumps(args_out_dict['args']),didSucceed,message)
		meta = args_out_dict.get('meta',{})
		if 'log_entry_durations' in meta:
			log_entry_durations = meta['log_entry_durations']
			log_entry_durations['app_duration'] = dt
			log_entry_durations['duration_enqueued'] = time_enqueued
			log_entry_durations['timestamp'] = next.utils.datetimeNow()
			ell.log( app_id+':ALG-DURATION', log_entry_durations  )
	else:
		return_value = (args_out_json,didSucceed,message)
	print '#### Finished %s,  time_enqueued=%s,  execution_time=%s ####' % (task_name,time_enqueued,dt)
	return return_value

def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.
        dir, _ = os.path.split(__file__)
        reference_dict,errs = verifier.load_doc('{}/myApp.yaml'.format(app_id, app_id),"apps/")
        if len(errs) > 0:
                raise Exception("App YAML format errors: \n{}".format(str(errs)))
        args_dict = verifier.verify(args_in_json, reference_dict['getStats']['args'])
        stat_id = args_dict['args'].get('stat_id','none')

        stat_args = args_dict['args']

        hash_object = hashlib.md5(stat_id+'_'+json.dumps(stat_args['params']))
        stat_uid = hash_object.hexdigest()
        stat_uid += '_' + exp_uid

        app = App_Wrapper(app_id, exp_uid, db, ell)
        cached_doc = app.butler.dashboard.get(uid=stat_uid)
        cached_response = None
        if (int(stat_args.get('force_recompute',0))==0) and (cached_doc is not None):
          delta_datetime = (next.utils.datetimeNow() - next.utils.str2datetime(cached_doc['timestamp']))
          if delta_datetime.seconds < next.constants.DASHBOARD_STALENESS_IN_SECONDS:
            cached_response = json.loads(cached_doc['data_dict'])
            if 'meta' not in cached_response:
              cached_response['meta']={}
            cached_response['meta']['cached'] = 1
            if delta_datetime.seconds/60<1:
                cached_response['meta']['last_dashboard_update'] = '<1 minute ago'
            else:
                cached_response['meta']['last_dashboard_update'] = str(delta_datetime.seconds/60)+' minutes ago'

        if cached_response==None:
            dashboard_string = 'apps.' + app_id + '.dashboard.Dashboard'
            dashboard_module = __import__(dashboard_string, fromlist=[''])
            dashboard = getattr(dashboard_module, 'MyAppDashboard')
            dashboard = dashboard(db, ell)
            stats_method = getattr(dashboard, stat_id)
            response,dt = next.utils.timeit(stats_method)(app,app.butler,**args_dict['args']['params'])

            save_dict = {'exp_uid':app.exp_uid,
                    'stat_uid':stat_uid,
                    'timestamp':next.utils.datetime2str(next.utils.datetimeNow()),
                    'data_dict':json.dumps(response)}
            app.butler.dashboard.set_many(uid=stat_uid,key_value_dict=save_dict)

            # update the admin timing with the timing of a getModel
            if hasattr(app, 'log_entry_durations'):
                app.log_entry_durations['app_duration'] = dt
                app.log_entry_durations['duration_enqueued'] = time_enqueued
                app.butler.ell.log(app.app_id+':ALG-DURATION', app.log_entry_durations)
        else:
            response = cached_response

        if DEBUG_ON:
            next.utils.debug_print('#### Finished Dashboard %s, time_enqueued=%s,  execution_time=%s ####' % (stat_id, time_enqueued, dt), color='white')
	return json.dumps(response), True, ''

def apply_sync_by_namespace(app_id, exp_uid, alg_id, alg_label, task_name, args, namespace, job_uid, enqueue_timestamp, time_limit):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	try:
		print '>>>>>>>> Starting namespace:%s,  job_uid=%s,  time_enqueued=%s <<<<<<<<<' % (namespace,job_uid,time_enqueued)
                # get stateless app
                next_app = next.utils.get_app(app_id, exp_uid, db, ell)
                target_manager = next_app.myApp.TargetManager
                next_alg = next.utils.get_app_alg(app_id, alg_id)
                butler = Butler(app_id, exp_uid, target_manager, db, ell, alg_label, alg_id)
		response,dt = next.utils.timeit(getattr(next_alg, task_name))(butler, args)
                log_entry_durations = { 'exp_uid':exp_uid,'alg_label':alg_label,'task':'daemonProcess','duration':dt }
                log_entry_durations.update(butler.algorithms.getDurations())
                log_entry_durations['app_duration'] = dt
                log_entry_durations['duration_enqueued'] = time_enqueued
                log_entry_durations['timestamp'] = next.utils.datetimeNow()
                ell.log( app_id+':ALG-DURATION', log_entry_durations)
		print '########## Finished namespace:%s,  job_uid=%s,  time_enqueued=%s,  execution_time=%s ##########' % (namespace,job_uid,time_enqueued,dt)
		return
	except Exception, error:
		exc_type, exc_value, exc_traceback = sys.exc_info()
                print "tasks Exception: {} {}".format(error, traceback.format_exc())
                traceback.print_tb(exc_traceback)

		# error = traceback.format_exc()
		# log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':next.utils.datetimeNow() }
		# ell.log( app_id+':APP-EXCEPTION', log_entry  )
		return None

# If celery isn't off, celery-wrap the functions so they can be called with apply_async
if next.constants.CELERY_ON:
        apply = app.task(apply)
        apply_dashboard = app.task(apply_dashboard)
        apply_sync_by_namespace = app.task(apply_sync_by_namespace)

<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.utils.functional import cached_property
from django.utils.translation import get_language, get_language_info
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from nextcloudappstore.core.models import App, Category

def app_description(request, id):
    app = get_object_or_404(App, id=id)
    return HttpResponse(app.description, content_type='text/plain')

class AppDetailView(DetailView):
    model = App
    template_name = 'app/detail.html'
    slug_field = 'id'
    slug_url_kwarg = 'id'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        context['latest_releases_by_platform_v'] = \
            self.object.latest_releases_by_platform_v()
        return context

class CategoryAppListView(ListView):
    model = App
    template_name = 'app/list.html'
    allow_empty = True

    def get_queryset(self):
        order_by = self.request.GET.get('order_by', 'last_modified')
        ordering = self.request.GET.get('ordering', 'desc')
        featured = self.request.GET.get('featured', False)
        maintainer = self.request.GET.get('maintainer', False)
        sort_columns = []

        allowed_order_by = {'name', 'last_modified'}
        if order_by in allowed_order_by:
            if order_by == 'name':
                order_by = 'translations__name'
            if ordering == 'desc':
                sort_columns.append('-' + order_by)
            else:
                sort_columns.append(order_by)

        lang = get_language_info(get_language())['code']
        category_id = self.kwargs['id']
        queryset = App.objects.search(self.search_terms, lang).order_by(
            *sort_columns)
        if category_id:
            queryset = queryset.filter(categories__id=category_id)
        if featured == "true":
            queryset = queryset.filter(featured=True)
        if maintainer:
            try:
                user = User.objects.get_by_natural_key(maintainer)
                queryset = queryset.filter(Q(owner=user) |
                                           Q(co_maintainers=user))
            except ObjectDoesNotExist:
                queryset = None
        return queryset

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        category_id = self.kwargs['id']
        if category_id:
            context['current_category'] = Category.objects.get(id=category_id)
        if self.search_terms:
            context['search_query'] = ' '.join(self.search_terms)
        context['auth_user'] = self.request.user
        context['logged_in'] = self.request.user.is_authenticated()
        return context

    @cached_property
    def search_terms(self):
        return self.request.GET.get('search', '').strip().split()
<commit_msg>Return empty list instead of None<commit_after>from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.utils.functional import cached_property
from django.utils.translation import get_language, get_language_info
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from nextcloudappstore.core.models import App, Category

def app_description(request, id):
    app = get_object_or_404(App, id=id)
    return HttpResponse(app.description, content_type='text/plain')

class AppDetailView(DetailView):
    model = App
    template_name = 'app/detail.html'
    slug_field = 'id'
    slug_url_kwarg = 'id'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        context['latest_releases_by_platform_v'] = \
            self.object.latest_releases_by_platform_v()
        return context

class CategoryAppListView(ListView):
    model = App
    template_name = 'app/list.html'
    allow_empty = True

    def get_queryset(self):
        order_by = self.request.GET.get('order_by', 'last_modified')
        ordering = self.request.GET.get('ordering', 'desc')
        featured = self.request.GET.get('featured', False)
        maintainer = self.request.GET.get('maintainer', False)
        sort_columns = []

        allowed_order_by = {'name', 'last_modified'}
        if order_by in allowed_order_by:
            if order_by == 'name':
                order_by = 'translations__name'
            if ordering == 'desc':
                sort_columns.append('-' + order_by)
            else:
                sort_columns.append(order_by)

        lang = get_language_info(get_language())['code']
        category_id = self.kwargs['id']
        queryset = App.objects.search(self.search_terms, lang).order_by(
            *sort_columns)
        if category_id:
            queryset = queryset.filter(categories__id=category_id)
        if featured == "true":
            queryset = queryset.filter(featured=True)
        if maintainer:
            try:
                user = User.objects.get_by_natural_key(maintainer)
                queryset = queryset.filter(Q(owner=user) |
                                           Q(co_maintainers=user))
            except ObjectDoesNotExist:
                queryset = []
        return queryset

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        category_id = self.kwargs['id']
        if category_id:
            context['current_category'] = Category.objects.get(id=category_id)
        if self.search_terms:
            context['search_query'] = ' '.join(self.search_terms)
        context['auth_user'] = self.request.user
        context['logged_in'] = self.request.user.is_authenticated()
        return context

    @cached_property
    def search_terms(self):
        return self.request.GET.get('search', '').strip().split()
<|endoftext|>


<commit_before>"""
Basic files needed by all Topographica programs.

This package should be self-contained, i.e., should not refer to any
other part of Topographica.  For instance, no file may include an
import statement like 'from topo.package.module import' or 'import
topo.package.module'.  This ensures that all of the packages outside
of this one are optional.

$Id$
"""
__version__='$Revision$'

__all__ = ['registry','simulator','parameter','utils','commandline','sheetview','gendocs','patterngenerator','sheet','boundingregion']
<commit_msg>Upated __all__ list<commit_after>"""
Basic files needed by all Topographica programs.

This package should be self-contained, i.e., should not refer to any
other part of Topographica.  For instance, no file may include an
import statement like 'from topo.package.module import' or 'import
topo.package.module'.  This ensures that all of the packages outside
of this one are optional.

$Id$
"""
__version__='$Revision$'

__all__ = ['boundingregion','commandline','connectionfield','distribution','gendocs','keyedlist','parameter','patternfns','patterngenerator','registry','sheet','sheetview','simulator','topoobject','utils']

<|endoftext|>


<commit_before>import topo
import param
import imagen

from topo import projection, responsefn, learningfn, transferfn, sheet, sparse
import topo.learningfn.optimized
import topo.transferfn.optimized
import topo.responsefn.optimized
import topo.sheet.optimized
import topo.transferfn.misc

from topo.submodel import Model, ArraySpec, order_projections # pyflakes:ignore (API import)
from topo.submodel.earlyvision import EarlyVisionModel

@Model.definition
class ModelGCAL(EarlyVisionModel):

    cortex_density=param.Number(default=47.0,bounds=(0,None),
        inclusive_bounds=(False,True),doc="""
        The nominal_density to use for V1.""")

    homeostasis = param.Boolean(default=True, doc="""
        Whether or not the homeostatic adaption should be applied in V1""")

    t_init = param.Number(default=0.15, doc="""
        The initial V1 threshold value. This value is static in the L and GCL models
        and adaptive in the AL and GCAL models.""")

    target_activity = param.Number(default=0.024,doc="""
        The target average activity for the homeostatic threshold mechanism.""")

    latexc_radius=param.Number(default=0.104,bounds=(0,None),doc="""
        Radius of the lateral excitatory bounds within V1.""")

    latinh_radius=param.Number(default=0.22917,bounds=(0,None),doc="""
        Radius of the lateral inhibitory bounds within V1.""")

    latexc_size=param.Number(default=0.05,bounds=(0,None),doc="""
        Size of the lateral excitatory connections within V1.""")

    latinh_size=param.Number(default=0.15,bounds=(0,None),doc="""
        Size of the lateral inhibitory connections within V1.""")

    aff_lr=param.Number(default=0.1,bounds=(0.0,None),doc="""
        Learning rate for the afferent projection(s) to V1.""")

    exc_lr=param.Number(default=0.0,bounds=(0.0,None),doc="""
        Learning rate for the lateral excitatory projection to V1.""")

    inh_lr=param.Number(default=0.3,bounds=(0.0,None),doc="""
        Learning rate for the lateral inhibitory projection to V1.""")

    aff_strength=param.Number(default=1.0,bounds=(0.0,None),doc="""
        Overall strength of the afferent projection to V1.""")

    exc_strength=param.Number(default=1.7,bounds=(0.0,None),doc="""
        Overall strength of the lateral excitatory projection to V1.""")

    inh_strength=param.Number(default=1.4,bounds=(0.0,None),doc="""
        Overall strength of the lateral inhibitory projection to V1.""")

    expand_sf_test_range=param.Boolean(default=False,doc="""
        By default, measure_sine_pref() measures SF at the sizes of RF
        used, for speed, but if expand_sf_test_range is True, it will
        test over a larger range, including half the size of the
        smallest and twice the size of the largest.""")

    def property_setup(self, properties):
        properties = super(ModelGCAL, self).property_setup(properties)
        "Specify weight initialization, response function, and learning function"

        projection.CFProjection.cf_shape=imagen.Disk(smoothing=0.0)
        projection.CFProjection.response_fn=responsefn.optimized.CFPRF_DotProduct_opt()
        projection.CFProjection.learning_fn=learningfn.optimized.CFPLF_Hebbian_opt()
        projection.CFProjection.weights_output_fns=[transferfn.optimized.CFPOF_DivisiveNormalizeL1_opt()]
        projection.SharedWeightCFProjection.response_fn=responsefn.optimized.CFPRF_DotProduct_opt()
        return properties

    def sheet_setup(self):
        sheets = super(ModelGCAL,self).sheet_setup()
        sheets['V1'] = [{}]
        return sheets

    @Model.SettlingCFSheet
    def V1(self, properties):
        return Model.SettlingCFSheet.params(
            tsettle=16,
            plastic=True,
            joint_norm_fn=topo.sheet.optimized.compute_joint_norm_totals_opt,
            output_fns=[transferfn.misc.HomeostaticResponse(t_init=self.t_init,
                                                            target_activity = self.target_activity,
                                                            learning_rate=0.01 if self.homeostasis else 0.0)],
            nominal_density=self.cortex_density,
            nominal_bounds=sheet.BoundingBox(radius=self.area/2.0))

    @Model.matchconditions('V1', 'V1_afferent')
    def V1_afferent_conditions(self, properties):
        return {'level': 'LGN'}

    @Model.CFProjection
    def V1_afferent(self, src_properties, dest_properties):
        sf_channel = src_properties['SF'] if 'SF' in src_properties else 1
        # Adjust delays so same measurement protocol can be used with and without gain control.
        LGN_V1_delay = 0.05 if self.gain_control else 0.10

        name=''
        if 'eye' in src_properties: name+=src_properties['eye']
        if 'opponent' in src_properties:
            name+=src_properties['opponent']+src_properties['surround']
        name+=('LGN'+src_properties['polarity']+'Afferent')
        if sf_channel>1: name+=('SF'+str(src_properties['SF']))

        gaussian_size = 2.0 * self.v1aff_radius *self.sf_spacing**(sf_channel-1)
        weights_generator = imagen.random.GaussianCloud(gaussian_size=gaussian_size)

        return [Model.CFProjection.params(
                delay=LGN_V1_delay+lag,
                dest_port=('Activity','JointNormalize','Afferent'),
                name= name if lag==0 else name+('Lag'+str(lag)),
                learning_rate=self.aff_lr,
                strength=self.aff_strength*(1.5 if self.gain_control else 1.0),
                weights_generator=weights_generator,
                nominal_bounds_template=sheet.BoundingBox(radius=
                                            self.v1aff_radius*self.sf_spacing**(sf_channel-1)))
                for lag in self['lags']]

    @Model.matchconditions('V1', 'lateral_excitatory')
    def lateral_excitatory_conditions(self, properties):
        return {'level': 'V1'}

    @Model.CFProjection
    def lateral_excitatory(self, src_properties, dest_properties):
        return Model.CFProjection.params(
            delay=0.05,
            name='LateralExcitatory',
            weights_generator=imagen.Gaussian(aspect_ratio=1.0, size=self.latexc_size),
            strength=self.exc_strength,
            learning_rate=self.exc_lr,
            nominal_bounds_template=sheet.BoundingBox(radius=self.latexc_radius))

    @Model.matchconditions('V1', 'lateral_inhibitory')
    def lateral_inhibitory_conditions(self, properties):
        return {'level': 'V1'}

    @Model.CFProjection
    def lateral_inhibitory(self, src_properties, dest_properties):
        return Model.CFProjection.params(
            delay=0.05,
            name='LateralInhibitory',
            weights_generator=imagen.random.GaussianCloud(gaussian_size=self.latinh_size),
            strength=-1.0*self.inh_strength,
            learning_rate=self.inh_lr,
            nominal_bounds_template=sheet.BoundingBox(radius=self.latinh_radius))

    def analysis_setup(self):
        # TODO: This is different in gcal.ty, stevens/gcal.ty and gcal_od.ty
        # And depends whether gain control is used or not
        import topo.analysis.featureresponses
        topo.analysis.featureresponses.FeatureMaps.selectivity_multiplier=2.0
        topo.analysis.featureresponses.FeatureCurveCommand.contrasts=[1, 10, 30, 50, 100]
        if 'dr' in self.dims:
            topo.analysis.featureresponses.MeasureResponseCommand.durations=[(max(self['lags'])+1)*1.0]
        if 'sf' in self.dims:
            from topo.analysis.command import measure_sine_pref
            sf_relative_sizes = [self.sf_spacing**(sf_channel-1) for sf_channel in self['SF']]
            wide_relative_sizes=[0.5*sf_relative_sizes[0]] + sf_relative_sizes + [2.0*sf_relative_sizes[-1]]
            relative_sizes=(wide_relative_sizes if self.expand_sf_test_range else sf_relative_sizes)
            #The default 2.4 spatial frequency value here is
            #chosen because it results in a sine grating with bars whose
            #width approximately matches the width of the Gaussian training
            #patterns, and thus the typical width of an ON stripe in one of the
            #receptive fields
            measure_sine_pref.frequencies = [2.4*s for s in relative_sizes]

@Model.definition
class ExamplesGCAL(ModelGCAL):
    """
    Reproduces the results of the legacy examples/gcal.ty file.
    """

    def __init__(self, **params):
        super(ExamplesGCAL, self).__init__(time_dependent=False, **params)
        if set(self.dims) != set([ 'xy', 'or']):
            raise Exception("ExamplesGCAL only reproducible for dims = ['xy', 'or']")

    def setup(self,setup_options=True):
        model = super(ExamplesGCAL, self).setup(setup_options)
        if setup_options is True or 'sheets' in setup_options:
            model.sheets.Retina.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+1.125))
            model.sheets.LGNOn.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+0.75))
            model.sheets.LGNOff.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+0.75))
            model.sheets.V1.update(nominal_density=48)
        if setup_options is True or 'projections' in setup_options:
            order_projections(model, ['afferent',
                                      'lateral_gain_control',
                                      ('V1_afferent', {'polarity':'On'}),
                                      ('V1_afferent', {'polarity':'Off'}),
                                      'lateral_excitatory',
                                      'lateral_inhibitory'])
        return model

<commit_msg>Removed unused import<commit_after>import topo
import param
import imagen

from topo import projection, responsefn, learningfn, transferfn, sheet
import topo.learningfn.optimized
import topo.transferfn.optimized
import topo.responsefn.optimized
import topo.sheet.optimized
import topo.transferfn.misc

from topo.submodel import Model, ArraySpec, order_projections # pyflakes:ignore (API import)
from topo.submodel.earlyvision import EarlyVisionModel

@Model.definition
class ModelGCAL(EarlyVisionModel):

    cortex_density=param.Number(default=47.0,bounds=(0,None),
        inclusive_bounds=(False,True),doc="""
        The nominal_density to use for V1.""")

    homeostasis = param.Boolean(default=True, doc="""
        Whether or not the homeostatic adaption should be applied in V1""")

    t_init = param.Number(default=0.15, doc="""
        The initial V1 threshold value. This value is static in the L and GCL models
        and adaptive in the AL and GCAL models.""")

    target_activity = param.Number(default=0.024,doc="""
        The target average activity for the homeostatic threshold mechanism.""")

    latexc_radius=param.Number(default=0.104,bounds=(0,None),doc="""
        Radius of the lateral excitatory bounds within V1.""")

    latinh_radius=param.Number(default=0.22917,bounds=(0,None),doc="""
        Radius of the lateral inhibitory bounds within V1.""")

    latexc_size=param.Number(default=0.05,bounds=(0,None),doc="""
        Size of the lateral excitatory connections within V1.""")

    latinh_size=param.Number(default=0.15,bounds=(0,None),doc="""
        Size of the lateral inhibitory connections within V1.""")

    aff_lr=param.Number(default=0.1,bounds=(0.0,None),doc="""
        Learning rate for the afferent projection(s) to V1.""")

    exc_lr=param.Number(default=0.0,bounds=(0.0,None),doc="""
        Learning rate for the lateral excitatory projection to V1.""")

    inh_lr=param.Number(default=0.3,bounds=(0.0,None),doc="""
        Learning rate for the lateral inhibitory projection to V1.""")

    aff_strength=param.Number(default=1.0,bounds=(0.0,None),doc="""
        Overall strength of the afferent projection to V1.""")

    exc_strength=param.Number(default=1.7,bounds=(0.0,None),doc="""
        Overall strength of the lateral excitatory projection to V1.""")

    inh_strength=param.Number(default=1.4,bounds=(0.0,None),doc="""
        Overall strength of the lateral inhibitory projection to V1.""")

    expand_sf_test_range=param.Boolean(default=False,doc="""
        By default, measure_sine_pref() measures SF at the sizes of RF
        used, for speed, but if expand_sf_test_range is True, it will
        test over a larger range, including half the size of the
        smallest and twice the size of the largest.""")

    def property_setup(self, properties):
        properties = super(ModelGCAL, self).property_setup(properties)
        "Specify weight initialization, response function, and learning function"

        projection.CFProjection.cf_shape=imagen.Disk(smoothing=0.0)
        projection.CFProjection.response_fn=responsefn.optimized.CFPRF_DotProduct_opt()
        projection.CFProjection.learning_fn=learningfn.optimized.CFPLF_Hebbian_opt()
        projection.CFProjection.weights_output_fns=[transferfn.optimized.CFPOF_DivisiveNormalizeL1_opt()]
        projection.SharedWeightCFProjection.response_fn=responsefn.optimized.CFPRF_DotProduct_opt()
        return properties

    def sheet_setup(self):
        sheets = super(ModelGCAL,self).sheet_setup()
        sheets['V1'] = [{}]
        return sheets

    @Model.SettlingCFSheet
    def V1(self, properties):
        return Model.SettlingCFSheet.params(
            tsettle=16,
            plastic=True,
            joint_norm_fn=topo.sheet.optimized.compute_joint_norm_totals_opt,
            output_fns=[transferfn.misc.HomeostaticResponse(t_init=self.t_init,
                                                            target_activity = self.target_activity,
                                                            learning_rate=0.01 if self.homeostasis else 0.0)],
            nominal_density=self.cortex_density,
            nominal_bounds=sheet.BoundingBox(radius=self.area/2.0))

    @Model.matchconditions('V1', 'V1_afferent')
    def V1_afferent_conditions(self, properties):
        return {'level': 'LGN'}

    @Model.CFProjection
    def V1_afferent(self, src_properties, dest_properties):
        sf_channel = src_properties['SF'] if 'SF' in src_properties else 1
        # Adjust delays so same measurement protocol can be used with and without gain control.
        LGN_V1_delay = 0.05 if self.gain_control else 0.10

        name=''
        if 'eye' in src_properties: name+=src_properties['eye']
        if 'opponent' in src_properties:
            name+=src_properties['opponent']+src_properties['surround']
        name+=('LGN'+src_properties['polarity']+'Afferent')
        if sf_channel>1: name+=('SF'+str(src_properties['SF']))

        gaussian_size = 2.0 * self.v1aff_radius *self.sf_spacing**(sf_channel-1)
        weights_generator = imagen.random.GaussianCloud(gaussian_size=gaussian_size)

        return [Model.CFProjection.params(
                delay=LGN_V1_delay+lag,
                dest_port=('Activity','JointNormalize','Afferent'),
                name= name if lag==0 else name+('Lag'+str(lag)),
                learning_rate=self.aff_lr,
                strength=self.aff_strength*(1.5 if self.gain_control else 1.0),
                weights_generator=weights_generator,
                nominal_bounds_template=sheet.BoundingBox(radius=
                                            self.v1aff_radius*self.sf_spacing**(sf_channel-1)))
                for lag in self['lags']]

    @Model.matchconditions('V1', 'lateral_excitatory')
    def lateral_excitatory_conditions(self, properties):
        return {'level': 'V1'}

    @Model.CFProjection
    def lateral_excitatory(self, src_properties, dest_properties):
        return Model.CFProjection.params(
            delay=0.05,
            name='LateralExcitatory',
            weights_generator=imagen.Gaussian(aspect_ratio=1.0, size=self.latexc_size),
            strength=self.exc_strength,
            learning_rate=self.exc_lr,
            nominal_bounds_template=sheet.BoundingBox(radius=self.latexc_radius))

    @Model.matchconditions('V1', 'lateral_inhibitory')
    def lateral_inhibitory_conditions(self, properties):
        return {'level': 'V1'}

    @Model.CFProjection
    def lateral_inhibitory(self, src_properties, dest_properties):
        return Model.CFProjection.params(
            delay=0.05,
            name='LateralInhibitory',
            weights_generator=imagen.random.GaussianCloud(gaussian_size=self.latinh_size),
            strength=-1.0*self.inh_strength,
            learning_rate=self.inh_lr,
            nominal_bounds_template=sheet.BoundingBox(radius=self.latinh_radius))

    def analysis_setup(self):
        # TODO: This is different in gcal.ty, stevens/gcal.ty and gcal_od.ty
        # And depends whether gain control is used or not
        import topo.analysis.featureresponses
        topo.analysis.featureresponses.FeatureMaps.selectivity_multiplier=2.0
        topo.analysis.featureresponses.FeatureCurveCommand.contrasts=[1, 10, 30, 50, 100]
        if 'dr' in self.dims:
            topo.analysis.featureresponses.MeasureResponseCommand.durations=[(max(self['lags'])+1)*1.0]
        if 'sf' in self.dims:
            from topo.analysis.command import measure_sine_pref
            sf_relative_sizes = [self.sf_spacing**(sf_channel-1) for sf_channel in self['SF']]
            wide_relative_sizes=[0.5*sf_relative_sizes[0]] + sf_relative_sizes + [2.0*sf_relative_sizes[-1]]
            relative_sizes=(wide_relative_sizes if self.expand_sf_test_range else sf_relative_sizes)
            #The default 2.4 spatial frequency value here is
            #chosen because it results in a sine grating with bars whose
            #width approximately matches the width of the Gaussian training
            #patterns, and thus the typical width of an ON stripe in one of the
            #receptive fields
            measure_sine_pref.frequencies = [2.4*s for s in relative_sizes]

@Model.definition
class ExamplesGCAL(ModelGCAL):
    """
    Reproduces the results of the legacy examples/gcal.ty file.
    """

    def __init__(self, **params):
        super(ExamplesGCAL, self).__init__(time_dependent=False, **params)
        if set(self.dims) != set([ 'xy', 'or']):
            raise Exception("ExamplesGCAL only reproducible for dims = ['xy', 'or']")

    def setup(self,setup_options=True):
        model = super(ExamplesGCAL, self).setup(setup_options)
        if setup_options is True or 'sheets' in setup_options:
            model.sheets.Retina.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+1.125))
            model.sheets.LGNOn.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+0.75))
            model.sheets.LGNOff.update(nominal_bounds=sheet.BoundingBox(radius=self.area/2.0+0.75))
            model.sheets.V1.update(nominal_density=48)
        if setup_options is True or 'projections' in setup_options:
            order_projections(model, ['afferent',
                                      'lateral_gain_control',
                                      ('V1_afferent', {'polarity':'On'}),
                                      ('V1_afferent', {'polarity':'Off'}),
                                      'lateral_excitatory',
                                      'lateral_inhibitory'])
        return model
<|endoftext|>