<commit_before># -*- coding: utf-8 -*-
from fractions import gcd
import networkx as nx
"""Algorithms for directed acyclic graphs (DAGs)."""
#    Copyright (C) 2006-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
__author__ = """\n""".join(['Aric Hagberg <aric.hagberg@gmail.com>',
                            'Dan Schult (dschult@colgate.edu)',
                            'Ben Edwards (bedwards@cs.unm.edu)'])
__all__ = ['descendants',
           'ancestors',
           'topological_sort',
           'topological_sort_recursive',
           'is_directed_acyclic_graph',
           'is_aperiodic']

def descendants(G, source):
    """Return all nodes reachable from `source` in G.

    Parameters
    ----------
    G : NetworkX DiGraph
    source : node in G

    Returns
    -------
    des : set()
       The descendants of source in G
    """
    if not G.has_node(source):
        raise nx.NetworkXError("The node %s is not in the graph." % source)
    des = set(nx.shortest_path_length(G, source=source).keys()) - set([source])
    return des

def ancestors(G, source):
    """Return all nodes having a path to `source` in G.

    Parameters
    ----------
    G : NetworkX DiGraph
    source : node in G

    Returns
    -------
    ancestors : set()
       The ancestors of source in G
    """
    if not G.has_node(source):
        raise nx.NetworkXError("The node %s is not in the graph." % source)
    anc = set(nx.shortest_path_length(G, target=source).keys()) - set([source])
    return anc

def is_directed_acyclic_graph(G):
    """Return True if the graph G is a directed acyclic graph (DAG) or
    False if not.

    Parameters
    ----------
    G : NetworkX graph
      A graph

    Returns
    -------
    is_dag : bool
       True if G is a DAG, false otherwise
    """
    if not G.is_directed():
        return False
    try:
        topological_sort(G)
        return True
    except nx.NetworkXUnfeasible:
        return False

def topological_sort(G,nbunch=None):
    """Return a list of nodes in topological sort order.

    A topological sort is a nonunique permutation of the nodes
    such that an edge from u to v implies that u appears before v in the
    topological sort order.

    Parameters
    ----------
    G : NetworkX digraph
       A directed graph

    nbunch : container of nodes (optional)
       Explore graph in specified order given in nbunch

    Raises
    ------
    NetworkXError
       Topological sort is defined for directed graphs only. If the
       graph G is undirected, a NetworkXError is raised.

    NetworkXUnfeasible
       If G is not a directed acyclic graph (DAG) no topological sort
       exists and a NetworkXUnfeasible exception is raised.

    Notes
    -----
    This algorithm is based on a description and proof in
    The Algorithm Design Manual [1]_ .

    See also
    --------
    is_directed_acyclic_graph

    References
    ----------
    .. [1] Skiena, S. S. The Algorithm Design Manual  (Springer-Verlag, 1998).
        http://www.amazon.com/exec/obidos/ASIN/0387948600/ref=ase_thealgorithmrepo/
    """
    if not G.is_directed():
        raise nx.NetworkXError(
                "Topological sort not defined on undirected graphs.")

    # nonrecursive version
    seen = set()
    order = []
    explored = set()

    if nbunch is None:
        nbunch = G.nodes_iter()
    for v in nbunch:     # process all vertices in G
        if v in explored:
            continue
        fringe = [v]   # nodes yet to look at
        while fringe:
            w = fringe[-1]  # depth first search
            if w in explored: # already looked down this branch
                fringe.pop()
                continue
            seen.add(w)     # mark as seen
            # Check successors for cycles and for new nodes
            new_nodes = []
            for n in G[w]:
                if n not in explored:
                    if n in seen: #CYCLE !!
                        raise nx.NetworkXUnfeasible("Graph contains a cycle.")
                    new_nodes.append(n)
            if new_nodes:   # Add new_nodes to fringe
                fringe.extend(new_nodes)
            else:           # No new nodes so w is fully explored
                explored.add(w)
                order.append(w)
                fringe.pop()    # done considering this node
    return list(reversed(order))

def topological_sort_recursive(G,nbunch=None):
    """Return a list of nodes in topological sort order.

    A topological sort is a nonunique permutation of the nodes such
    that an edge from u to v implies that u appears before v in the
    topological sort order.

    Parameters
    ----------
    G : NetworkX digraph

    nbunch : container of nodes (optional)
       Explore graph in specified order given in nbunch

    Raises
    ------
    NetworkXError
       Topological sort is defined for directed graphs only. If the
       graph G is undirected, a NetworkXError is raised.

    NetworkXUnfeasible
        If G is not a directed acyclic graph (DAG) no topological sort
        exists and a NetworkXUnfeasible exception is raised.

    Notes
    -----
    This is a recursive version of topological sort.

    See also
    --------
    topological_sort
    is_directed_acyclic_graph

    """
    if not G.is_directed():
        raise nx.NetworkXError(
            "Topological sort not defined on undirected graphs.")

    def _dfs(v):
        ancestors.add(v)

        for w in G[v]:
            if w in ancestors:
                raise nx.NetworkXUnfeasible("Graph contains a cycle.")

            if w not in explored:
                _dfs(w)

        ancestors.remove(v)
        explored.add(v)
        order.append(v)

    ancestors = set()
    explored = set()
    order = []

    if nbunch is None:
        nbunch = G.nodes_iter()

    for v in nbunch:
        if v not in explored:
            _dfs(v)

    return list(reversed(order))

def is_aperiodic(G):
    """Return True if G is aperiodic.

    A directed graph is aperiodic if there is no integer k > 1 that
    divides the length of every cycle in the graph.

    Parameters
    ----------
    G : NetworkX DiGraph
      Graph

    Returns
    -------
    aperiodic : boolean
      True if the graph is aperiodic False otherwise

    Raises
    ------
    NetworkXError
      If G is not directed

    Notes
    -----
    This uses the method outlined in [1]_, which runs in O(m) time
    given m edges in G. Note that a graph is not aperiodic if it is
    acyclic as every integer trivial divides length 0 cycles.

    References
    ----------
    .. [1] Jarvis, J. P.; Shier, D. R. (1996),
       Graph-theoretic analysis of finite Markov chains,
       in Shier, D. R.; Wallenius, K. T., Applied Mathematical Modeling:
       A Multidisciplinary Approach, CRC Press.
    """
    if not G.is_directed():
        raise nx.NetworkXError("is_aperiodic not defined for undirected graphs")

    s = next(G.nodes_iter())
    levels = {s:0}
    this_level = [s]
    g = 0
    l = 1
    while this_level:
        next_level = []
        for u in this_level:
            for v in G[u]:
                if v in levels: # Non-Tree Edge
                    g = gcd(g, levels[u]-levels[v] + 1)
                else: # Tree Edge
                    next_level.append(v)
                    levels[v] = l
        this_level = next_level
        l += 1
    if len(levels)==len(G): #All nodes in tree
        return g==1
    else:
        return g==1 and nx.is_aperiodic(G.subgraph(set(G)-set(levels)))
<commit_msg>ENH: add reverse kwarg for topo sort<commit_after># -*- coding: utf-8 -*-
from fractions import gcd
import networkx as nx
"""Algorithms for directed acyclic graphs (DAGs)."""
#    Copyright (C) 2006-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
__author__ = """\n""".join(['Aric Hagberg <aric.hagberg@gmail.com>',
                            'Dan Schult (dschult@colgate.edu)',
                            'Ben Edwards (bedwards@cs.unm.edu)'])
__all__ = ['descendants',
           'ancestors',
           'topological_sort',
           'topological_sort_recursive',
           'is_directed_acyclic_graph',
           'is_aperiodic']

def descendants(G, source):
    """Return all nodes reachable from `source` in G.

    Parameters
    ----------
    G : NetworkX DiGraph
    source : node in G

    Returns
    -------
    des : set()
       The descendants of source in G
    """
    if not G.has_node(source):
        raise nx.NetworkXError("The node %s is not in the graph." % source)
    des = set(nx.shortest_path_length(G, source=source).keys()) - set([source])
    return des

def ancestors(G, source):
    """Return all nodes having a path to `source` in G.

    Parameters
    ----------
    G : NetworkX DiGraph
    source : node in G

    Returns
    -------
    ancestors : set()
       The ancestors of source in G
    """
    if not G.has_node(source):
        raise nx.NetworkXError("The node %s is not in the graph." % source)
    anc = set(nx.shortest_path_length(G, target=source).keys()) - set([source])
    return anc

def is_directed_acyclic_graph(G):
    """Return True if the graph G is a directed acyclic graph (DAG) or
    False if not.

    Parameters
    ----------
    G : NetworkX graph
      A graph

    Returns
    -------
    is_dag : bool
       True if G is a DAG, false otherwise
    """
    if not G.is_directed():
        return False
    try:
        topological_sort(G)
        return True
    except nx.NetworkXUnfeasible:
        return False

def topological_sort(G, nbunch=None, reverse=False):
    """Return a list of nodes in topological sort order.

    A topological sort is a nonunique permutation of the nodes
    such that an edge from u to v implies that u appears before v in the
    topological sort order.

    Parameters
    ----------
    G : NetworkX digraph
       A directed graph

    nbunch : container of nodes (optional)
       Explore graph in specified order given in nbunch

    reverse : bool, optional
        Return postorder instead of preorder if True.

    Raises
    ------
    NetworkXError
       Topological sort is defined for directed graphs only. If the
       graph G is undirected, a NetworkXError is raised.

    NetworkXUnfeasible
       If G is not a directed acyclic graph (DAG) no topological sort
       exists and a NetworkXUnfeasible exception is raised.

    Notes
    -----
    This algorithm is based on a description and proof in
    The Algorithm Design Manual [1]_ .

    See also
    --------
    is_directed_acyclic_graph

    References
    ----------
    .. [1] Skiena, S. S. The Algorithm Design Manual  (Springer-Verlag, 1998).
        http://www.amazon.com/exec/obidos/ASIN/0387948600/ref=ase_thealgorithmrepo/
    """
    if not G.is_directed():
        raise nx.NetworkXError(
                "Topological sort not defined on undirected graphs.")

    # nonrecursive version
    seen = set()
    order = []
    explored = set()

    if nbunch is None:
        nbunch = G.nodes_iter()
    for v in nbunch:     # process all vertices in G
        if v in explored:
            continue
        fringe = [v]   # nodes yet to look at
        while fringe:
            w = fringe[-1]  # depth first search
            if w in explored: # already looked down this branch
                fringe.pop()
                continue
            seen.add(w)     # mark as seen
            # Check successors for cycles and for new nodes
            new_nodes = []
            for n in G[w]:
                if n not in explored:
                    if n in seen: #CYCLE !!
                        raise nx.NetworkXUnfeasible("Graph contains a cycle.")
                    new_nodes.append(n)
            if new_nodes:   # Add new_nodes to fringe
                fringe.extend(new_nodes)
            else:           # No new nodes so w is fully explored
                explored.add(w)
                order.append(w)
                fringe.pop()    # done considering this node
    if reverse:
        return order
    else:
        return list(reversed(order))

def topological_sort_recursive(G, nbunch=None, reverse=False):
    """Return a list of nodes in topological sort order.

    A topological sort is a nonunique permutation of the nodes such
    that an edge from u to v implies that u appears before v in the
    topological sort order.

    Parameters
    ----------
    G : NetworkX digraph

    nbunch : container of nodes (optional)
       Explore graph in specified order given in nbunch

    reverse : bool, optional
        Return postorder instead of preorder if True.

    Raises
    ------
    NetworkXError
       Topological sort is defined for directed graphs only. If the
       graph G is undirected, a NetworkXError is raised.

    NetworkXUnfeasible
        If G is not a directed acyclic graph (DAG) no topological sort
        exists and a NetworkXUnfeasible exception is raised.

    Notes
    -----
    This is a recursive version of topological sort.

    See also
    --------
    topological_sort
    is_directed_acyclic_graph

    """
    if not G.is_directed():
        raise nx.NetworkXError(
            "Topological sort not defined on undirected graphs.")

    def _dfs(v):
        ancestors.add(v)

        for w in G[v]:
            if w in ancestors:
                raise nx.NetworkXUnfeasible("Graph contains a cycle.")

            if w not in explored:
                _dfs(w)

        ancestors.remove(v)
        explored.add(v)
        order.append(v)

    ancestors = set()
    explored = set()
    order = []

    if nbunch is None:
        nbunch = G.nodes_iter()

    for v in nbunch:
        if v not in explored:
            _dfs(v)

    if reverse:
        return order
    else:
        return list(reversed(order))

def is_aperiodic(G):
    """Return True if G is aperiodic.

    A directed graph is aperiodic if there is no integer k > 1 that
    divides the length of every cycle in the graph.

    Parameters
    ----------
    G : NetworkX DiGraph
      Graph

    Returns
    -------
    aperiodic : boolean
      True if the graph is aperiodic False otherwise

    Raises
    ------
    NetworkXError
      If G is not directed

    Notes
    -----
    This uses the method outlined in [1]_, which runs in O(m) time
    given m edges in G. Note that a graph is not aperiodic if it is
    acyclic as every integer trivial divides length 0 cycles.

    References
    ----------
    .. [1] Jarvis, J. P.; Shier, D. R. (1996),
       Graph-theoretic analysis of finite Markov chains,
       in Shier, D. R.; Wallenius, K. T., Applied Mathematical Modeling:
       A Multidisciplinary Approach, CRC Press.
    """
    if not G.is_directed():
        raise nx.NetworkXError("is_aperiodic not defined for undirected graphs")

    s = next(G.nodes_iter())
    levels = {s:0}
    this_level = [s]
    g = 0
    l = 1
    while this_level:
        next_level = []
        for u in this_level:
            for v in G[u]:
                if v in levels: # Non-Tree Edge
                    g = gcd(g, levels[u]-levels[v] + 1)
                else: # Tree Edge
                    next_level.append(v)
                    levels[v] = l
        this_level = next_level
        l += 1
    if len(levels)==len(G): #All nodes in tree
        return g==1
    else:
        return g==1 and nx.is_aperiodic(G.subgraph(set(G)-set(levels)))
<|endoftext|>


<commit_before>"""Remote Python Debugger (pdb wrapper)."""

__author__ = "Bertrand Janin <b@janin.com>"
__version__ = "0.1.5"

import pdb
import socket
import sys

class Rpdb(pdb.Pdb):

    def __init__(self, addr="127.0.0.1", port=4444):
        """Initialize the socket and initialize pdb."""

        # Backup stdin and stdout before replacing them by the socket handle
        self.old_stdout = sys.stdout
        self.old_stdin = sys.stdin

        # Open a 'reusable' socket to let the webapp reload on the same port
        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
        self.skt.bind((addr, port))
        self.skt.listen(1)

        # Writes to stdout are forbidden in mod_wsgi environments
        try:
            sys.stderr.write("pdb is running on %s:%d\n" % self.skt.getsockname())
        except IOError:
            pass

        (clientsocket, address) = self.skt.accept()
        handle = clientsocket.makefile('rw')
        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
        sys.stdout = sys.stdin = handle

    def shutdown(self):
        """Revert stdin and stdout, close the socket."""
        sys.stdout = self.old_stdout
        sys.stdin = self.old_stdin
        self.skt.close()

    def do_continue(self, arg):
        """Clean-up and do underlying continue."""
        try:
            return pdb.Pdb.do_continue(self, arg)
        finally:
            self.shutdown()

    do_c = do_cont = do_continue

    def do_quit(self, arg):
        """Clean-up and do underlying quit."""
        try:
            return pdb.Pdb.do_quit(self, arg)
        finally:
            self.shutdown()

    do_q = do_exit = do_quit

    def do_EOF(self, arg):
        """Clean-up and do underlying quit."""
        try:
            return pdb.Pdb.do_EOF(self, arg)
        finally:
            self.shutdown()

def set_trace(addr="127.0.0.1", port=4444):
    """Wrapper function to keep the same import x; x.set_trace() interface.

    We catch all the possible exceptions from pdb and cleanup.

    """
    debugger = Rpdb(addr=addr, port=port)
    try:
        debugger.set_trace(sys._getframe().f_back)
    except Exception:
        print(Exception)
<commit_msg>Rectify comment cut/paste error.<commit_after>"""Remote Python Debugger (pdb wrapper)."""

__author__ = "Bertrand Janin <b@janin.com>"
__version__ = "0.1.5"

import pdb
import socket
import sys

class Rpdb(pdb.Pdb):

    def __init__(self, addr="127.0.0.1", port=4444):
        """Initialize the socket and initialize pdb."""

        # Backup stdin and stdout before replacing them by the socket handle
        self.old_stdout = sys.stdout
        self.old_stdin = sys.stdin

        # Open a 'reusable' socket to let the webapp reload on the same port
        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
        self.skt.bind((addr, port))
        self.skt.listen(1)

        # Writes to stdout are forbidden in mod_wsgi environments
        try:
            sys.stderr.write("pdb is running on %s:%d\n" % self.skt.getsockname())
        except IOError:
            pass

        (clientsocket, address) = self.skt.accept()
        handle = clientsocket.makefile('rw')
        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
        sys.stdout = sys.stdin = handle

    def shutdown(self):
        """Revert stdin and stdout, close the socket."""
        sys.stdout = self.old_stdout
        sys.stdin = self.old_stdin
        self.skt.close()

    def do_continue(self, arg):
        """Clean-up and do underlying continue."""
        try:
            return pdb.Pdb.do_continue(self, arg)
        finally:
            self.shutdown()

    do_c = do_cont = do_continue

    def do_quit(self, arg):
        """Clean-up and do underlying quit."""
        try:
            return pdb.Pdb.do_quit(self, arg)
        finally:
            self.shutdown()

    do_q = do_exit = do_quit

    def do_EOF(self, arg):
        """Clean-up and do underlying EOF."""
        try:
            return pdb.Pdb.do_EOF(self, arg)
        finally:
            self.shutdown()

def set_trace(addr="127.0.0.1", port=4444):
    """Wrapper function to keep the same import x; x.set_trace() interface.

    We catch all the possible exceptions from pdb and cleanup.

    """
    debugger = Rpdb(addr=addr, port=port)
    try:
        debugger.set_trace(sys._getframe().f_back)
    except Exception:
        print(Exception)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

from django import forms
from django.utils.translation import (
    ugettext_lazy,
    ugettext as _,
)
from django.forms.widgets import Select
from django.core.exceptions import ValidationError
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit
from crispy_forms import layout as crispy

from corehq.apps.app_manager.dbaccessors import (
    get_brief_apps_in_domain,
    get_version_build_id,
)
from corehq.apps.app_manager.exceptions import BuildNotFoundException
from corehq.apps.hqwebapp.crispy import HQFormHelper
from corehq.apps.hqwebapp import crispy as hqcrispy
from custom.icds.models import (
    HostedCCZ,
    HostedCCZLink,
)

class HostedCCZLinkForm(forms.ModelForm):
    class Meta:
        model = HostedCCZLink
        exclude = ('domain',)

    def __init__(self, domain, *args, **kwargs):
        super(HostedCCZLinkForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper(self)
        save_button_text = _('Update') if self.instance.pk else _('Create')
        self.helper.layout.append(Submit('save', save_button_text))
        if self.instance.pk:
            del self.fields['password']
        else:
            self.fields['password'].widget = forms.PasswordInput()
        if self.instance.pk:
            self.helper.layout.append(Submit('delete', _('Delete')))
        self.helper.layout = crispy.Fieldset(_("CCZ Hosting Link"), self.helper.layout)
        self.fields['identifier'].widget.attrs.update({'class': 'text-lowercase'})
        self.instance.domain = domain

class HostedCCZForm(forms.Form):
    link_id = forms.ChoiceField(label=ugettext_lazy("Link"), choices=(), required=True)
    app_id = forms.ChoiceField(label=ugettext_lazy("Application"), choices=(), required=True)
    version = forms.IntegerField(label=ugettext_lazy('Version'), required=True, widget=Select(choices=[]))
    profile_id = forms.CharField(label=ugettext_lazy('Application Profile'),
                                 required=False, widget=Select(choices=[]))
    file_name = forms.CharField(label=ugettext_lazy("CCZ File Name"), required=False)
    note = forms.CharField(required=False, widget=forms.Textarea(attrs={'rows': 3, 'cols': 15}))
    status = forms.ChoiceField(label=ugettext_lazy("Status"),
                               choices=(
                                   ('', 'Select Status'),
                                   (HostedCCZ.PENDING, 'Pending'),
                                   (HostedCCZ.BUILDING, 'Building'),
                                   (HostedCCZ.FAILED, 'Failed'),
                                   (HostedCCZ.COMPLETED, 'Completed')),
                               required=False,
                               help_text=ugettext_lazy("Applicable for search only"))

    def __init__(self, request, domain, email, *args, **kwargs):
        self.domain = domain
        self.email = email
        super(HostedCCZForm, self).__init__(*args, **kwargs)
        self.fields['link_id'].choices = self.link_choices()
        self.fields['app_id'].choices = self.app_id_choices()
        self.helper = HQFormHelper()
        if request.GET.get('app_id'):
            self.fields['app_id'].initial = request.GET.get('app_id')
        if request.GET.get('link_id'):
            self.fields['link_id'].initial = request.GET.get('link_id')
        if request.GET.get('status'):
            self.fields['status'].initial = request.GET.get('status')
        self.helper.layout = crispy.Layout(
            crispy.Field('link_id', css_class="hqwebapp-select2", id="link-id-select"),
            crispy.Field('app_id', css_class="hqwebapp-select2", id='app-id-search-select'),
            crispy.Field('version', id='version-input'),
            crispy.Field('profile_id', id='app-profile-id-input'),
            crispy.Field('file_name'),
            crispy.Field('note'),
            crispy.Field('status'),
            hqcrispy.FormActions(
                crispy.ButtonHolder(
                    crispy.Button('search', ugettext_lazy("Search"), data_bind="click: search"),
                    crispy.Button('clear', ugettext_lazy("Clear"), data_bind="click: clear"),
                    Submit('submit', ugettext_lazy("Create"))
                )
            )
        )

    def app_id_choices(self):
        choices = [(None, _('Select Application'))]
        for app in get_brief_apps_in_domain(self.domain):
            choices.append((app.id, app.name))
        return choices

    def link_choices(self):
        choices = [(None, _('Select Link'))]
        for link in HostedCCZLink.objects.filter(domain=self.domain):
            choices.append((link.id, link.identifier))
        return choices

    def _version_exists(self):
        return bool(get_version_build_id(self.domain, self.cleaned_data['app_id'],
                                         self.cleaned_data['version']))

    def clean(self):
        if self.cleaned_data.get('app_id') and self.cleaned_data.get('version'):
            try:
                self._version_exists()
            except BuildNotFoundException as e:
                self.add_error('version', e)

    def save(self):
        try:
            HostedCCZ(
                link_id=self.cleaned_data['link_id'], app_id=self.cleaned_data['app_id'],
                version=self.cleaned_data['version'], profile_id=self.cleaned_data['profile_id'],
                file_name=self.cleaned_data['file_name'],
                note=self.cleaned_data['note'],
            ).save(email=self.email)
        except ValidationError as e:
            return False, ','.join(e.messages)
        return True, None
<commit_msg>translate options<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

from django import forms
from django.utils.translation import (
    ugettext_lazy,
    ugettext as _,
)
from django.forms.widgets import Select
from django.core.exceptions import ValidationError
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit
from crispy_forms import layout as crispy

from corehq.apps.app_manager.dbaccessors import (
    get_brief_apps_in_domain,
    get_version_build_id,
)
from corehq.apps.app_manager.exceptions import BuildNotFoundException
from corehq.apps.hqwebapp.crispy import HQFormHelper
from corehq.apps.hqwebapp import crispy as hqcrispy
from custom.icds.models import (
    HostedCCZ,
    HostedCCZLink,
)

class HostedCCZLinkForm(forms.ModelForm):
    class Meta:
        model = HostedCCZLink
        exclude = ('domain',)

    def __init__(self, domain, *args, **kwargs):
        super(HostedCCZLinkForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper(self)
        save_button_text = _('Update') if self.instance.pk else _('Create')
        self.helper.layout.append(Submit('save', save_button_text))
        if self.instance.pk:
            del self.fields['password']
        else:
            self.fields['password'].widget = forms.PasswordInput()
        if self.instance.pk:
            self.helper.layout.append(Submit('delete', _('Delete')))
        self.helper.layout = crispy.Fieldset(_("CCZ Hosting Link"), self.helper.layout)
        self.fields['identifier'].widget.attrs.update({'class': 'text-lowercase'})
        self.instance.domain = domain

class HostedCCZForm(forms.Form):
    link_id = forms.ChoiceField(label=ugettext_lazy("Link"), choices=(), required=True)
    app_id = forms.ChoiceField(label=ugettext_lazy("Application"), choices=(), required=True)
    version = forms.IntegerField(label=ugettext_lazy('Version'), required=True, widget=Select(choices=[]))
    profile_id = forms.CharField(label=ugettext_lazy('Application Profile'),
                                 required=False, widget=Select(choices=[]))
    file_name = forms.CharField(label=ugettext_lazy("CCZ File Name"), required=False)
    note = forms.CharField(required=False, widget=forms.Textarea(attrs={'rows': 3, 'cols': 15}))
    status = forms.ChoiceField(label=ugettext_lazy("Status"),
                               choices=(
                                   ('', ugettext_lazy('Select Status')),
                                   (HostedCCZ.PENDING, ugettext_lazy('Pending')),
                                   (HostedCCZ.BUILDING, ugettext_lazy('Building')),
                                   (HostedCCZ.FAILED, ugettext_lazy('Failed')),
                                   (HostedCCZ.COMPLETED, ugettext_lazy('Completed'))),
                               required=False,
                               help_text=ugettext_lazy("Applicable for search only"))

    def __init__(self, request, domain, email, *args, **kwargs):
        self.domain = domain
        self.email = email
        super(HostedCCZForm, self).__init__(*args, **kwargs)
        self.fields['link_id'].choices = self.link_choices()
        self.fields['app_id'].choices = self.app_id_choices()
        self.helper = HQFormHelper()
        if request.GET.get('app_id'):
            self.fields['app_id'].initial = request.GET.get('app_id')
        if request.GET.get('link_id'):
            self.fields['link_id'].initial = request.GET.get('link_id')
        if request.GET.get('status'):
            self.fields['status'].initial = request.GET.get('status')
        self.helper.layout = crispy.Layout(
            crispy.Field('link_id', css_class="hqwebapp-select2", id="link-id-select"),
            crispy.Field('app_id', css_class="hqwebapp-select2", id='app-id-search-select'),
            crispy.Field('version', id='version-input'),
            crispy.Field('profile_id', id='app-profile-id-input'),
            crispy.Field('file_name'),
            crispy.Field('note'),
            crispy.Field('status'),
            hqcrispy.FormActions(
                crispy.ButtonHolder(
                    crispy.Button('search', ugettext_lazy("Search"), data_bind="click: search"),
                    crispy.Button('clear', ugettext_lazy("Clear"), data_bind="click: clear"),
                    Submit('submit', ugettext_lazy("Create"))
                )
            )
        )

    def app_id_choices(self):
        choices = [(None, _('Select Application'))]
        for app in get_brief_apps_in_domain(self.domain):
            choices.append((app.id, app.name))
        return choices

    def link_choices(self):
        choices = [(None, _('Select Link'))]
        for link in HostedCCZLink.objects.filter(domain=self.domain):
            choices.append((link.id, link.identifier))
        return choices

    def _version_exists(self):
        return bool(get_version_build_id(self.domain, self.cleaned_data['app_id'],
                                         self.cleaned_data['version']))

    def clean(self):
        if self.cleaned_data.get('app_id') and self.cleaned_data.get('version'):
            try:
                self._version_exists()
            except BuildNotFoundException as e:
                self.add_error('version', e)

    def save(self):
        try:
            HostedCCZ(
                link_id=self.cleaned_data['link_id'], app_id=self.cleaned_data['app_id'],
                version=self.cleaned_data['version'], profile_id=self.cleaned_data['profile_id'],
                file_name=self.cleaned_data['file_name'],
                note=self.cleaned_data['note'],
            ).save(email=self.email)
        except ValidationError as e:
            return False, ','.join(e.messages)
        return True, None
<|endoftext|>


<commit_before>#!/usr/bin/python3
#python script to quickly plot geographical grids
# Author Roelof Rietbroek (http://wobbly.earth)
# initial version 5 December 2017
# License: see file LICENSE (MIT)
#part of speedplot
import sys
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
from optparse import OptionParser
from netCDF4 import Dataset
import numpy as np
import re
def main(argv):

#set up command line options
    usage=argv[0]+ " [Options] NCFILE\n"\
            +"Plot geographical grids from netcdf file (NCFILE) and possibly points\n"

    parser=OptionParser(usage)
    parser.add_option("-V","--Var",metavar="VAR[N]",default='z[0]',type="string",help="specify the netcdf variable name and layer to plot ")
    # parser.add_option("-l","--legend",type="string",metavar="LEGEND",help="Make a legend for the plot by specifying LEGEND as: Curve1/Another Curve/..")
    parser.add_option('-c','--columns',type="string",metavar="COLS",help="only print specific columns from the files e.g. COLS: 2/6/8, the default prints all columns in all files")
    parser.add_option('-z',"--zlabel",type="string",default="",help="Specify a label to put on the color (z) axis")
    parser.add_option('-r',"--range",type="string",metavar="ZMIN,ZMAX",help="set range of colorbar")
    parser.add_option("--cmap",default="viridis",metavar="COLORMAP",help="Set colormap name")
    parser.add_option('-t',"--title",type="string",help="Add a title to the plot")
    parser.add_option('-o',"--output", metavar="IMAGE", type="string",help="Output the plot to an image rather than a dynamic viewer. Suffices (e.g. .pdf, .eps, .svg, .png) are automatically detected from IMAGE but must be supported by the matplotlib backend)")
    parser.add_option('--transparency',action="store_true",help="Set the background to be transparent")
    #parser.add_option('-m',"--multiply",type="string",metavar="SCALE", help="multiply the grid SCALE")

    #parser.add_option('-g',"--grid",action="store_true",help="show grid on the plot")
    parser.add_option('-p',"--projection",type="string",metavar="PROJ",default='glob180',help="choose projection:\n"\
            +"glob(global equidist centered on 0 deg meridian), glob180 (centered on 180 deg meridian)")
    parser.add_option('-s',"--symbols",type="string",metavar="LONLATVALUEFILE",help="Plot symbols at lon lat positions with color proportional to value")

    (options, args) = parser.parse_args()
    fids=[]

    if not args:
        grdfile=[]
        if not options.symbols:
            print("Error: no netcdf file or lonlat file given",file=sys.stderr)
            parser.print_help()
            sys.exit(1)
    else:
        grdfile=args[:][0]

    if grdfile:
     #open netcdf data file
        print(grdfile)
        ncid=Dataset(grdfile,'r')

        #try to read in longitude and latitude
        for latname in ["y","latitude","Latitude"]:
            try:
                lat=ncid[latname][:]
                break
            except:
                 pass
        if not lat.any():
            print("couldn't find a suitable latitude variable",file=sys.stderr)
            sys.exit(1)

        for lonname in ["x","longitude","Longitude"]:
            try:
                lon=ncid[lonname][:]
                break
            except:
                 pass
        if not lon.any():
            print("couldn't find a suitable longitude variable",file=sys.stderr)
            sys.exit(1)

        #load gridvariable
        try:
            ztpl=re.findall('(.*?)\[(.*?)\]',options.Var)
            if ztpl:
                name=ztpl[0][0]
                dim=int(ztpl[0][1])
            else:
                name=options.Var
                dim=0
            z=ncid[name][:]

            if z.ndim ==3:
                z=z[dim]

        except:
            print("cannot load %s"%options.Var,file=sys.stderr)
            sys.exit(1)
        ncid.close()

    if options.symbols:

        #read station name and position
        plon=[]
        plat=[]
        pval=[]
        fid=open(options.symbols,'r')
        for ln in fid.readlines():
            lspl=ln.split()
            plon.append(float(lspl[0]))
            plat.append(float(lspl[1]))
            if len(lspl) ==3:
                pval.append(float(lspl[2]))
        fid.close()

    if options.projection == 'glob':
        #make a global plot centered on the 0 meridian
        m = Basemap(projection='cyl',llcrnrlat=-90,urcrnrlat=90, llcrnrlon=-180,urcrnrlon=180,resolution='c')
        #possibly shift data when going from 0 to 360
        print(max(lon),file=sys.stderr)
        if max(lon) > 180:
            print("Sorry, glob projection with this grid requires a shifting of grid data (not yet implemented",file=sys.stderr)
            sys.exit(1)
    elif options.projection == 'glob180':
        m = Basemap(projection='cyl',llcrnrlat=-90,urcrnrlat=90, llcrnrlon=0,urcrnrlon=360,resolution='c')
        if options.symbols:
            for i in range(len(plon)):
                if plon[i] < 0:
                    plon[i]=360+plon[i]

   #make a base plot for the antarctic contintent

    #m = Basemap(projection='spaeqd',boundinglat=-65,lon_0=180,lat_0=-90,resolution='h')
    # m.shadedrelief()

    if grdfile:
        #plot the gridded data
        m.imshow(z,extent=[min(lon), max(lon),min(lat),max(lat)])

    m.drawcoastlines()

    if options.symbols:
        print(min(plon),min(plat),min(pval),file=sys.stderr)

        if pval:
    # #plot symbols on top
            if options.range:
                spltrng=options.range.split(',')
                m.scatter(plon,plat,c=pval,vmin=float(spltrng[0]),vmax=float(spltrng[1]),cmap=options.cmap)
            else:
                m.scatter(plon,plat,c=pval)
            m.colorbar(label=options.zlabel)

        else:
            m.scatter(plon,plat)

	#add title
    if options.title:
        plt.title(options.title)

#print or show the figure
    if options.output:
        plt.savefig(options.output,bbox_inches='tight',transparent=options.transparency)
    else:
        plt.show()

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>add scaling and better handle colorbar<commit_after>#!/usr/bin/python3
#python script to quickly plot geographical grids
# Author Roelof Rietbroek (http://wobbly.earth)
# initial version 5 December 2017
# License: see file LICENSE (MIT)
#part of speedplot
import sys
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
from optparse import OptionParser
from netCDF4 import Dataset
import numpy as np
import re
def main(argv):

#set up command line options
    usage=argv[0]+ " [Options] NCFILE\n"\
            +"Plot geographical grids from netcdf file (NCFILE) and possibly points\n"

    parser=OptionParser(usage)
    parser.add_option("-V","--Var",metavar="VAR[N]",default='z[0]',type="string",help="specify the netcdf variable name and layer to plot ")
    # parser.add_option("-l","--legend",type="string",metavar="LEGEND",help="Make a legend for the plot by specifying LEGEND as: Curve1/Another Curve/..")
    parser.add_option('-z',"--zlabel",type="string",default="",help="Specify a label to put on the color (z) axis")
    parser.add_option('-r',"--range",type="string",metavar="ZMIN,ZMAX",help="set range of colorbar")
    parser.add_option("--cmap",default="viridis",metavar="COLORMAP",help="Set colormap name")
    parser.add_option('-t',"--title",type="string",help="Add a title to the plot")
    parser.add_option('-o',"--output", metavar="IMAGE", type="string",help="Output the plot to an image rather than a dynamic viewer. Suffices (e.g. .pdf, .eps, .svg, .png) are automatically detected from IMAGE but must be supported by the matplotlib backend)")
    parser.add_option('--transparency',action="store_true",help="Set the background to be transparent")
    parser.add_option('-m',"--multiply",type="float",default=1.0,metavar="SCALE", help="multiply the grid and symbols values with SCALE")

    #parser.add_option('-g',"--grid",action="store_true",help="show grid on the plot")
    parser.add_option('-p',"--projection",type="string",metavar="PROJ",default='glob180',help="choose projection:\n"\
            +"glob(global equidist centered on 0 deg meridian), glob180 (centered on 180 deg meridian)")
    parser.add_option('-s',"--symbols",type="string",metavar="LONLATVALUEFILE",help="Plot symbols at lon lat positions with color proportional to value")

    (options, args) = parser.parse_args()
    fids=[]

    if not args:
        grdfile=[]
        if not options.symbols:
            print("Error: no netcdf file or lonlat file given",file=sys.stderr)
            parser.print_help()
            sys.exit(1)
    else:
        grdfile=args[:][0]

    if grdfile:
     #open netcdf data file
        print(grdfile)
        ncid=Dataset(grdfile,'r')

        #try to read in longitude and latitude
        for latname in ["y","latitude","Latitude"]:
            try:
                lat=ncid[latname][:]
                break
            except:
                 pass
        if not lat.any():
            print("couldn't find a suitable latitude variable",file=sys.stderr)
            sys.exit(1)

        for lonname in ["x","longitude","Longitude"]:
            try:
                lon=ncid[lonname][:]
                break
            except:
                 pass
        if not lon.any():
            print("couldn't find a suitable longitude variable",file=sys.stderr)
            sys.exit(1)

        #load gridvariable
        try:
            ztpl=re.findall('(.*?)\[(.*?)\]',options.Var)
            if ztpl:
                name=ztpl[0][0]
                dim=int(ztpl[0][1])
            else:
                name=options.Var
                dim=0
            z=options.multiply*ncid[name][:]

            if z.ndim ==3:
                z=z[dim]

        except:
            print("cannot load %s"%options.Var,file=sys.stderr)
            sys.exit(1)
        ncid.close()

    if options.symbols:

        #read station name and position
        plon=[]
        plat=[]
        pval=[]
        fid=open(options.symbols,'r')
        for ln in fid.readlines():
            lspl=ln.split()
            plon.append(float(lspl[0]))
            plat.append(float(lspl[1]))
            if len(lspl) ==3:
                pval.append(float(lspl[2]))
        fid.close()

    if options.projection == 'glob':
        #make a global plot centered on the 0 meridian
        m = Basemap(projection='cyl',llcrnrlat=-90,urcrnrlat=90, llcrnrlon=-180,urcrnrlon=180,resolution='c')
        #possibly shift data when going from 0 to 360
        print(max(lon),file=sys.stderr)
        if max(lon) > 180:
            print("Sorry, glob projection with this grid requires a shifting of grid data (not yet implemented",file=sys.stderr)
            sys.exit(1)
    elif options.projection == 'glob180':
        m = Basemap(projection='cyl',llcrnrlat=-90,urcrnrlat=90, llcrnrlon=0,urcrnrlon=360,resolution='c')
        if options.symbols:
            for i in range(len(plon)):
                if plon[i] < 0:
                    plon[i]=360+plon[i]

   #make a base plot for the antarctic contintent

    #m = Basemap(projection='spaeqd',boundinglat=-65,lon_0=180,lat_0=-90,resolution='h')
    # m.shadedrelief()

    if grdfile:
        #plot the gridded data
        if options.range:

            spltrng=[float(x) for x in options.range.split(',')]
            m.imshow(z,extent=[min(lon), max(lon),min(lat),max(lat)],vmin=spltrng[0],vmax=spltrng[1],cmap=options.cmap)

        else:
            m.imshow(z,extent=[min(lon), max(lon),min(lat),max(lat)],cmap=options.cmap)
        cbar=m.colorbar(label=options.zlabel)

    m.drawcoastlines()

    if options.symbols:
        # print(min(plon),min(plat),min(pval),file=sys.stderr)

        if pval:
    # #plot symbols on top
            if options.range:
                spltrng=[float(x) for x in options.range.split(',')]
                m.scatter(plon,plat,c=pval,vmin=spltrng[0],vmax=spltrng[1],cmap=options.cmap)
            else:
                m.scatter(plon,plat,c=pval)
            if not cbar:
                m.colorbar(label=options.zlabel,cmap=options.cmap)

        else:
            m.scatter(plon,plat)

	#add title
    if options.title:
        plt.title(options.title)

#print or show the figure
    if options.output:
        plt.savefig(options.output,bbox_inches='tight',transparent=options.transparency)
    else:
        plt.show()

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>print('Hello World')
<commit_msg>add function<commit_after>str = '{"a":12,"b":3,"cdasfewafdsafefdsaf":"dsafe"}'
index = 0
length = len(str)

def parse_json():
    parse_object()
    print "success"

def parse_object():
    global index, length, str
    while index < length:
        char = str[index+1]
        if char == '}':
            index+=1
            print "object end, pos:" , index
            return True
        elif char == '"':
            parse_attribute()
        elif ' ':
            index+=1
        elif ',':
            index+=1
        else:
            raise SyntaxError(index)

def parse_attribute():
    global index, length, str
    index+=1
    parse_string()
    if not str[index] == ':':
        raise SyntaxError(index)
    index+=1
    parse_value()

def parse_value():
    global index, length, str
    index+=1
    while True:
        if str[index] == ',':
            index-=1
            break;
        if str[index] == '}':
            index-=1
            break;
        index+=1

def parse_string():
    global index, length, str
    index+=1;
    while True:
        if str[index] == '"':
            index+=1
            break;
        index+=1

parse_json()<|endoftext|>


<commit_before>from django.contrib import admin

from .models import Email

class EmailAdmin(admin.ModelAdmin):
    exclude = ('targetted_users', 'is_sent')

admin.site.register(Email, EmailAdmin)
<commit_msg>Customize how fields on Email are displayed while adding & editing<commit_after>from django.contrib import admin

from .models import Email

class EmailAdmin(admin.ModelAdmin):
    readonly_fields = ('targetted_users', 'is_sent',)
    add_fieldsets = (
        (None, {
            'fields': ('subject', 'message', 'post'),
        }),
    )

    def get_fieldsets(self, request, obj=None):
        if not obj:
            return self.add_fieldsets
        return super(EmailAdmin, self).get_fieldsets(request, obj)

admin.site.register(Email, EmailAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from nimp.commands._cis_command import *
from nimp.utilities.ue3         import *
from nimp.utilities.ue4         import *
from nimp.utilities.deployment  import *
from nimp.utilities.file_mapper import *

#-------------------------------------------------------------------------------
class CisBuildCommand(CisCommand):
    abstract = 0
    def __init__(self):
        CisCommand.__init__(self,
                            'cis-build',
                            'Build UE3 or UE4 executable and publishes it to a shared directory')

    #---------------------------------------------------------------------------
    def cis_configure_arguments(self, env, parser):
        parser.add_argument('-r',
                            '--revision',
                            help    = 'Current revision',
                            metavar = '<revision>')

        parser.add_argument('-p',
                            '--platform',
                            help    = 'platforms to build',
                            metavar = '<platform>')

        parser.add_argument('-c',
                            '--configuration',
                            help    = 'configuration to build',
                            metavar = '<configuration>')

        return True

    #---------------------------------------------------------------------------
    def _cis_run(self, env):
        env.generate_version_file = True

        if not env.publish_only:
            log_notification(log_prefix() + "Building game…")

            # Unreal Engine 3
            if hasattr(env, 'project_type') and env.project_type is 'UE3':
                if not ue3_build(env):
                    return False

        log_notification(log_prefix() + "Publishing Binaries…")
        files_to_publish = env.map_files().to(env.publish_binaries).load_set("binaries")
        if not all_map(robocopy, files_to_publish()):
            return False

        log_notification(log_prefix() + "Publishing symbols…")
        symbols_to_publish = env.map_files().load_set("symbols")
        if not upload_symbols(env, symbols_to_publish()):
            return False

        return True

<commit_msg>Fully remove the “publish_only” feature.<commit_after># -*- coding: utf-8 -*-

from nimp.commands._cis_command import *
from nimp.utilities.ue3         import *
from nimp.utilities.ue4         import *
from nimp.utilities.deployment  import *
from nimp.utilities.file_mapper import *

#-------------------------------------------------------------------------------
class CisBuildCommand(CisCommand):
    abstract = 0
    def __init__(self):
        CisCommand.__init__(self,
                            'cis-build',
                            'Build UE3 or UE4 executable and publishes it to a shared directory')

    #---------------------------------------------------------------------------
    def cis_configure_arguments(self, env, parser):
        parser.add_argument('-r',
                            '--revision',
                            help    = 'Current revision',
                            metavar = '<revision>')

        parser.add_argument('-p',
                            '--platform',
                            help    = 'platforms to build',
                            metavar = '<platform>')

        parser.add_argument('-c',
                            '--configuration',
                            help    = 'configuration to build',
                            metavar = '<configuration>')

        return True

    #---------------------------------------------------------------------------
    def _cis_run(self, env):
        env.generate_version_file = True

        log_notification(log_prefix() + "Building game…")

        # Unreal Engine 3
        if hasattr(env, 'project_type') and env.project_type is 'UE3':
            if not ue3_build(env):
                return False

        # Unreal Engine 4
        if hasattr(env, 'project_type') and env.project_type is 'UE4':
            log_error(log_prefix() + "cis-build is deprecated for UE4 projects")
            return False

        log_notification(log_prefix() + "Publishing Binaries…")
        files_to_publish = env.map_files().to(env.publish_binaries).load_set("binaries")
        if not all_map(robocopy, files_to_publish()):
            return False

        log_notification(log_prefix() + "Publishing symbols…")
        symbols_to_publish = env.map_files().load_set("symbols")
        if not upload_symbols(env, symbols_to_publish()):
            return False

        return True

<|endoftext|>


<commit_before>
def sufficient_statistics(data_train, Y):
    """ input: Y contains labellings for all y^(n)
    returns counts, over the entire dataset, of the form $ \sum_n \#c\in(\mathbf{x}^{(n)},\mathbf{y^{(n)}} $
    """
    grad_f = np.zeros((data_train.n_labels * data_train.X.shape[0] + data_train.n_labels **2), dtype=util.dtype_for_arrays)
    # noting grad_f, but this is one of the terms of grad_f, really
    for n in range(data_train.N):
        for t in range(data_train.object_size[n]):
            grad_f[data_train.unaries[n][t, Y[n][t]]] += 1
        for t in range(1, data_train.object_size[n]):
            grad_f[data_train.binaries[Y[n][t-1], Y[n][t]]] += 1
    return grad_f

import prepare_from_data
import prepare_from_data_chain
def sample_given_f(data_train, f):
    """ input: log potentials f
    returns a sample Y over the dataset such that for all n, $y^{(n)} \sim p( \cdot | x^{(n)}, f)"""
    marginals = prepare_from_data.posterior_marginals(f, data_train, prepare_from_data_chain.marginals_function)
    # can't use posterior_marginals_test: we want marginals over the training data ! this implies we need to carry around data_train
    Y = []
    for (n, marginals_n) in enumerate(marginals):
        y_n = np.empty(data_train.object_size[n], dtype=np.int16)
        for t in range(data_train.object_size[n]):
            y_n[t] = np.random.choice(data_train.n_labels, p=marginals_n[t,:])
        Y.append(y_n)
    return Y

def expected_statistics(data_train, f):
    """ computes the expected count of average statistics, ie the term d log Z/df. this is one of the two terms of
    d log lik (D_train) / df"""
    n_samples = 50 # computing expectation over fixed number of samples -- surely there are cleverer schemes
    accumulated_statistics = np.zeros_like(f, dtype=util.dtype_for_arrays)
    for n in range(n_samples):
        accumulated_statistics += sufficient_statistics(data_train, sample_given_f(data_train, f))
    return accumulated_statistics/n_samples

def grad_likelihood(data_train, f):
    return sufficient_statistics(data_train, data_train.Y) - expected_statistics(data_train, f)

def compute_Kinv_f(f, L):
    return L.solve_cholesky_lower(f) # K^-1 f = (L L^T)^-1 f

def grad_prior(f, L):
    return -L.solve_cholesky_lower(f)

def grad_posterior(f, L, data_train):
    return grad_likelihood(data_train, f) + grad_prior(f, L)

import gc
def func(x, L, ll_train, logger):
    x = util.dtype_for_arrays(x) # sometimes x will come in float64
    log_prior_val = util.dtype_for_arrays(-1/2) * x.dot(compute_Kinv_f(x, L)) # log p(f) = -1/2 f^T K^-1 f + const (which we disregard)
    log_posterior_val = -ll_train(x) - log_prior_val
    logger.debug("objective function value %.5g" % log_posterior_val)
    gc.collect()
    return log_posterior_val

def grad(x, L, data_train, logger):
    x = util.dtype_for_arrays(x) # sometimes x will come in float64
    return - grad_posterior(x, L, data_train)

import logging
import time
import sys
def obtain_logger(config):
    logger = logging.getLogger('pyGPstruct') # reuse logger if existing
    logger.setLevel(2)
    logger.propagate = False
    logger.handlers = [] # remove any existing handlers
    # create formatter
    formatter = logging.Formatter('%(asctime)sZ - %(levelname)s - %(message)s')
    formatter.converter=time.gmtime # will use GMT timezone for output, hence the Z
    # console log handler
    ch = logging.StreamHandler(stream=sys.stdout)
    #ch.setLevel(logging.INFO) # avoid clogging console (more info in file)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    fh = logging.FileHandler(config['result_prefix'] + "log")
    fh.setFormatter(formatter)
    logger.addHandler(fh)

    return logger

import scipy
import kernels
import numpy as np
import pickle
import util
import os

def run(config):
    np.random.seed = config["seed"]
    # prepare data-dependent functions
    n_features = {'basenp' : 6438, 'chunking' : 29764, 'segmentation' : 1386, 'japanesene' : 102799}[config['task']]
    n_data = {'basenp' : 300, 'chunking' : 100, 'segmentation' : 36, 'japanesene' : 100}[config['task']]
    n_data_train = {'basenp' : 150, 'chunking' : 50, 'segmentation' : 20, 'japanesene' : 50}[config['task']]
    fold_indices = np.loadtxt(config['pygpstruct_location'] + '/data/datasplit.n_data=%s.txt' % n_data, dtype=np.int16) - 1 # need -1 because doing +1 inside prepare_data_chain

    common_arguments = {
    'data_indices_train' : fold_indices[config['fold'],:n_data_train],
    'data_indices_test' : fold_indices[config['fold'],n_data_train:],
    'data_folder' : config['pygpstruct_location'] + '/data/' + config['task'],
    'native_implementation': True,
    }

    import dataset_chain
    data_train = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_train'])
    data_test = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_test'])
    # will need to use inside prepare_from_data.posterior_marginals

    os.makedirs(config['result_prefix'], exist_ok=True)
    logger=obtain_logger(config)

    import prepare_from_data_chain
    (ll_train,
    posterior_marginals_test,
    compute_error_nlm,
    ll_test,
    average_marginals,
    write_marginals,
    read_marginals,
    n_labels,
    X_train,
    X_test) = prepare_from_data_chain.prepare_from_data_chain(task=config['task'],
                                                           logger=logger,
                                                           **common_arguments
                                               )

    # prepare K, L=cholK, and then func and grad for the minimizer
    kernel = kernels.kernel_exponential_ard
    lhp = {'unary': np.log(1), 'binary': np.log(1), 'jitter': np.log(1e-4),
                           'variances' : +1 * np.ones((n_features))}
    L = kernels.compute_lower_chol_k(kernel, lhp, X_train, n_labels)
    func_L = lambda x : func(x,L, ll_train, logger)
    grad_L = lambda x : grad(x,L, data_train, logger)
    u = np.random.randn(data_train.n_labels * data_train.X.shape[0] + data_train.n_labels **2)
    x0 = L.dot(u.astype(util.dtype_for_arrays))
    res = scipy.optimize.minimize(func_L, x0, method=config['optimization.method'],
                                  jac=grad_L, options=config['optimization.options'])
    f_star_mean = kernels.compute_k_star_T_k_inv(kernel, lhp, X_train, X_test, n_labels, L).dot(res.x)
    marginals_f_star = posterior_marginals_test(f_star_mean)
    error_metrics = compute_error_nlm(marginals_f_star)

    result = {'f' : res.x,
                         'accuracy' : error_metrics[0],
                         'nlm' : error_metrics[1]}
    result.update(config)
    with open(config['result_prefix'] + 'map.pickle', 'wb') as result_file:
        pickle.dump(result, result_file)
    logger.info(result)

# TODO add already written tests here<commit_msg>tests<commit_after>
def sufficient_statistics(data_train, Y):
    """ input: Y contains labellings for all y^(n)
    returns counts, over the entire dataset, of the form $ \sum_n \#c\in(\mathbf{x}^{(n)},\mathbf{y^{(n)}} $
    """
    grad_f = np.zeros((data_train.n_labels * data_train.X.shape[0] + data_train.n_labels **2), dtype=util.dtype_for_arrays)
    # noting grad_f, but this is one of the terms of grad_f, really
    for n in range(data_train.N):
        for t in range(data_train.object_size[n]):
            grad_f[data_train.unaries[n][t, Y[n][t]]] += 1
        for t in range(1, data_train.object_size[n]):
            grad_f[data_train.binaries[Y[n][t-1], Y[n][t]]] += 1
    return grad_f

import prepare_from_data
import prepare_from_data_chain
def sample_given_f(data_train, f):
    """ input: log potentials f
    returns a sample Y over the dataset such that for all n, $y^{(n)} \sim p( \cdot | x^{(n)}, f)"""
    marginals = prepare_from_data.posterior_marginals(f, data_train, prepare_from_data_chain.marginals_function)
    # can't use posterior_marginals_test: we want marginals over the training data ! this implies we need to carry around data_train
    Y = []
    for (n, marginals_n) in enumerate(marginals):
        y_n = np.empty(data_train.object_size[n], dtype=np.int16)
        for t in range(data_train.object_size[n]):
            y_n[t] = np.random.choice(data_train.n_labels, p=marginals_n[t,:])
        Y.append(y_n)
    return Y

def expected_statistics(data_train, f):
    """ computes the expected count of average statistics, ie the term d log Z/df. this is one of the two terms of
    d log lik (D_train) / df"""
    n_samples = 50 # computing expectation over fixed number of samples -- surely there are cleverer schemes
    accumulated_statistics = np.zeros_like(f, dtype=util.dtype_for_arrays)
    for n in range(n_samples):
        accumulated_statistics += sufficient_statistics(data_train, sample_given_f(data_train, f))
    return accumulated_statistics/n_samples

def grad_likelihood(data_train, f):
    return sufficient_statistics(data_train, data_train.Y) - expected_statistics(data_train, f)

def compute_Kinv_f(f, L):
    return L.solve_cholesky_lower(f) # K^-1 f = (L L^T)^-1 f

def grad_prior(f, L):
    return -L.solve_cholesky_lower(f)

def grad_posterior(f, L, data_train):
    return grad_likelihood(data_train, f) + grad_prior(f, L)

import gc
def func(x, L, ll_train, logger):
    x = util.dtype_for_arrays(x) # sometimes x will come in float64
    log_prior_val = util.dtype_for_arrays(-1/2) * x.dot(compute_Kinv_f(x, L)) # log p(f) = -1/2 f^T K^-1 f + const (which we disregard)
    log_posterior_val = -ll_train(x) - log_prior_val
    logger.debug("objective function value %.5g" % log_posterior_val)
    gc.collect()
    return log_posterior_val

def grad(x, L, data_train, logger):
    x = util.dtype_for_arrays(x) # sometimes x will come in float64
    return - grad_posterior(x, L, data_train)

import logging
import time
import sys
def obtain_logger(config):
    logger = logging.getLogger('pyGPstruct') # reuse logger if existing
    logger.setLevel(2)
    logger.propagate = False
    logger.handlers = [] # remove any existing handlers
    # create formatter
    formatter = logging.Formatter('%(asctime)sZ - %(levelname)s - %(message)s')
    formatter.converter=time.gmtime # will use GMT timezone for output, hence the Z
    # console log handler
    ch = logging.StreamHandler(stream=sys.stdout)
    #ch.setLevel(logging.INFO) # avoid clogging console (more info in file)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if 'result_prefix' in config:
        fh = logging.FileHandler(config['result_prefix'] + "log")
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    return logger

import scipy
import kernels
import numpy as np
import pickle
import util
import os

def run(config):
    np.random.seed = config["seed"]
    # prepare data-dependent functions
    n_features = {'basenp' : 6438, 'chunking' : 29764, 'segmentation' : 1386, 'japanesene' : 102799}[config['task']]
    n_data = {'basenp' : 300, 'chunking' : 100, 'segmentation' : 36, 'japanesene' : 100}[config['task']]
    n_data_train = {'basenp' : 150, 'chunking' : 50, 'segmentation' : 20, 'japanesene' : 50}[config['task']]
    fold_indices = np.loadtxt(config['pygpstruct_location'] + '/data/datasplit.n_data=%s.txt' % n_data, dtype=np.int16) - 1 # need -1 because doing +1 inside prepare_data_chain

    common_arguments = {
    'data_indices_train' : fold_indices[config['fold'],:n_data_train],
    'data_indices_test' : fold_indices[config['fold'],n_data_train:],
    'data_folder' : config['pygpstruct_location'] + '/data/' + config['task'],
    'native_implementation': True,
    }

    import dataset_chain
    data_train = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_train'])
    data_test = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_test'])
    # will need to use inside prepare_from_data.posterior_marginals

    os.makedirs(config['result_prefix'], exist_ok=True)
    logger=obtain_logger(config)

    import prepare_from_data_chain
    (ll_train,
    posterior_marginals_test,
    compute_error_nlm,
    ll_test,
    average_marginals,
    write_marginals,
    read_marginals,
    n_labels,
    X_train,
    X_test) = prepare_from_data_chain.prepare_from_data_chain(task=config['task'],
                                                           logger=logger,
                                                           **common_arguments
                                               )

    # prepare K, L=cholK, and then func and grad for the minimizer
    #kernel = kernels.kernel_exponential_ard
    #lhp = {'unary': np.log(1), 'binary': np.log(1), 'jitter': np.log(1e-4), 'variances' : +1 * np.ones((n_features))}
    kernel = kernels.kernel_linear
    lhp = {'unary': np.log(1), 'binary': np.log(1), 'jitter': np.log(1e-4)}
    L = kernels.compute_lower_chol_k(kernel, lhp, X_train, n_labels)
    func_L = lambda x : func(x,L, ll_train, logger)
    grad_L = lambda x : grad(x,L, data_train, logger)

    # how to choose x0 ?
    if True:
        # read off stored experiment results
        with open('/xvdb/results-2015-06-23/pygpstruct_50k.task=%(task)s.hp_sampling_mode=None.parameterization=ub.fold=%(fold)g.hp_sampling_thinning=1000/state.pickle'
                  % config, 'rb') as file:
            x0 = pickle.load(file)['current_f']
    else:
        # pick random position
        u = np.random.randn(data_train.n_labels * data_train.X.shape[0] + data_train.n_labels **2)
        x0 = L.dot(u.astype(util.dtype_for_arrays))

    res = scipy.optimize.minimize(func_L, x0, method=config['optimization.method'],
                                  jac=grad_L, options=config['optimization.options'])
    f_star_mean = kernels.compute_k_star_T_k_inv(kernel, lhp, X_train, X_test, n_labels, L).dot(res.x)
    marginals_f_star = posterior_marginals_test(f_star_mean)
    error_metrics = compute_error_nlm(marginals_f_star)

    result = {'f' : res.x,
                         'accuracy' : error_metrics[0],
                         'nlm' : error_metrics[1]}
    result.update(config)
    with open(config['result_prefix'] + 'map.pickle', 'wb') as result_file:
        pickle.dump(result, result_file)
    logger.info(result)

# TESTS
if __name__ == "__main__":

    # TEST setup, copied from run() ------------
    config = {'task': 'segmentation', 'fold':0, 'pygpstruct_location': '/xvdb/pygpstruct/'}

    n_features = {'basenp' : 6438, 'chunking' : 29764, 'segmentation' : 1386, 'japanesene' : 102799}[config['task']]
    n_data = {'basenp' : 300, 'chunking' : 100, 'segmentation' : 36, 'japanesene' : 100}[config['task']]
    n_data_train = {'basenp' : 150, 'chunking' : 50, 'segmentation' : 20, 'japanesene' : 50}[config['task']]
    fold_indices = np.loadtxt(config['pygpstruct_location'] + '/data/datasplit.n_data=%s.txt' % n_data, dtype=np.int16) - 1 # need -1 because doing +1 inside prepare_data_chain

    common_arguments = {
    'data_indices_train' : fold_indices[config['fold'],:n_data_train],
    'data_indices_test' : fold_indices[config['fold'],n_data_train:],
    'data_folder' : config['pygpstruct_location'] + '/data/' + config['task'],
    'native_implementation': True,
    }

    import dataset_chain
    data_train = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_train'])
    data_test = dataset_chain.dataset_chain(task = config['task'], data_folder=common_arguments['data_folder'], sub_indices=common_arguments['data_indices_test'])
    # will need to use inside prepare_from_data.posterior_marginals

    logger=obtain_logger(config)
    logger.setLevel(logging.INFO)

    import prepare_from_data_chain
    (ll_train,
    posterior_marginals_test,
    compute_error_nlm,
    ll_test,
    average_marginals,
    write_marginals,
    read_marginals,
    n_labels,
    X_train,
    X_test) = prepare_from_data_chain.prepare_from_data_chain(task=config['task'],
                                                           logger=logger,
                                                           **common_arguments
                                               )

    # prepare K, L=cholK, and then func and grad for the minimizer
    #kernel = kernels.kernel_exponential_ard
    #lhp = {'unary': np.log(1), 'binary': np.log(1), 'jitter': np.log(1e-4), 'variances' : +1 * np.ones((n_features))}
    kernel = kernels.kernel_linear
    lhp = {'unary': np.log(1), 'binary': np.log(1), 'jitter': np.log(1e-4)}
    L = kernels.compute_lower_chol_k(kernel, lhp, X_train, n_labels)
    func_L = lambda x : func(x,L, ll_train, logger)
    grad_L = lambda x : grad(x,L, data_train, logger)

    u = np.random.randn(data_train.n_labels * data_train.X.shape[0] + data_train.n_labels **2)
    x0 = L.dot(u.astype(util.dtype_for_arrays))
    # END TEST setup, copied from run() ------------

    # TEST
    # suppose all f's corresponding to binary factors are equal, say 0.
    # then these two values should be equal:
    # - #edges in data
    # - expectation of #all edges over dataset
    np.testing.assert_almost_equal((data_train.n_points - data_train.N),
                                   expected_statistics(data_train, f)[-9:].sum())

    # TEST: manual gradient check in one direction
    # I can do a manual check in one single direction, to have an idea of orders of magnitude: take a step in direction eps * np.ones(), and compare:

    eps = 1e-7
    np.testing.assert_almost_equal(func_L(x0 + eps) - func_L(x0), # first terms of Taylor expansion
                                   grad_L(x0).dot(eps*np.ones_like(x0)), # gradient value
                                   decimal=3)
    # gradient works ! at least in that direction.

    # TEST: gradient check
    # check_grad will make a move of amplitude eps into each direction (ie along each dimension of x0), and compute the difference between the finite-diff approx (f(x + eps) - f(x) / eps), and the gradient in that direction
    assert(scipy.optimize.check_grad(func_L, grad_L, x0, epsilon=0.1) < 1e5)
    # this is long, because it iterates over the number of dimensions of f-space, eg 3522 in the present configuration<|endoftext|>


<commit_before>import os
import numpy as np
import nibabel as nb
import pandas as pd
import glob
from nltools.simulator import Simulator
from nltools.data import Brain_Data, Adjacency, Groupby
from nltools.data import threshold
from nltools.mask import create_sphere
from sklearn.metrics import pairwise_distances
import matplotlib
matplotlib.use('TkAgg')

def test_brain_data(tmpdir):
    sim = Simulator()
    r = 10
    sigma = 1
    y = [0, 1]
    n_reps = 3
    output_dir = str(tmpdir)
    sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)

    shape_3d = (91, 109, 91)
    shape_2d = (6, 238955)
    y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None,index_col=None).T
    flist = glob.glob(str(tmpdir.join('centered*.nii.gz')))

    # Test load list
    dat = Brain_Data(data=flist,Y=y)

    # Test load file
    assert Brain_Data(flist[0])

    # Test to_nifti
    d = dat.to_nifti()
    assert d.shape[0:3] == shape_3d

    # Test load nibabel
    assert Brain_Data(d)

    # Test shape
    assert dat.shape() == shape_2d

    # Test Mean
    assert dat.mean().shape()[0] == shape_2d[1]

    # Test Std
    assert dat.std().shape()[0] == shape_2d[1]

    # Test add
    new = dat + dat
    assert new.shape() == shape_2d

    # Test subtract
    new = dat - dat
    assert new.shape() == shape_2d

    # Test multiply
    new = dat * dat
    assert new.shape() == shape_2d

    # Test Iterator
    x = [x for x in dat]
    assert len(x)==len(dat)
    assert len(x[0].data.shape)==1

    # # Test T-test
    out = dat.ttest()
    assert out['t'].shape()[0]==shape_2d[1]

    # # # Test T-test - permutation method
    # out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1})
    # assert out['t'].shape()[0]==shape_2d[1]

    # Test Regress
    dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)),'X1':np.array(dat.Y).flatten()},index=None)
    out = dat.regress()
    assert out['beta'].shape() == (2,shape_2d[1])

    # Test indexing
    assert out['t'][1].shape()[0] == shape_2d[1]

    # Test threshold
    i=1
    tt = threshold(out['t'][i], out['p'][i], .05)
    assert isinstance(tt,Brain_Data)

    # Test write
    dat.write(os.path.join(str(tmpdir.join('test_write.nii'))))
    assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii'))))

    # Test append
    assert dat.append(dat).shape()[0]==shape_2d[0]*2

    # Test distance
    distance = dat.distance(method='euclidean')
    assert isinstance(distance,Adjacency)
    assert distance.square_shape()[0]==shape_2d[0]

    # Test predict
    stats = dat.predict(algorithm='svm', cv_dict={'type': 'kfolds','n_folds': 2, 'n':len(dat.Y)}, plot=False,**{'kernel':"linear"})

    # Support Vector Regression, with 5 fold cross-validation with Platt Scaling
    # This will output probabilities of each class
    stats = dat.predict(algorithm='svm', cv_dict=None, plot=False,**{'kernel':'linear','probability':True})

    assert isinstance(stats['weight_map'],Brain_Data)
    # Logistic classificiation, with 5 fold stratified cross-validation.

    stats = dat.predict(algorithm='logistic', cv_dict={'type': 'kfolds','n_folds': 5, 'n':len(dat.Y)}, plot=False)
    assert isinstance(stats['weight_map'],Brain_Data)

    # Ridge classificiation, with 5 fold between-subject cross-validation, where data for each subject is held out together.
    stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None,plot=False)
    assert isinstance(stats['weight_map'],Brain_Data)

    # Test Similarity
    r = dat.similarity(stats['weight_map'])
    assert len(r)==shape_2d[0]
    r2 = dat.similarity(stats['weight_map'].to_nifti())
    assert len(r2)==shape_2d[0]

    # Test apply_mask - might move part of this to test mask suite
    s1 = create_sphere([41, 64, 55], radius=10)
    assert isinstance(s1,nb.Nifti1Image)
    s2 = Brain_Data(s1)
    masked_dat = dat.apply_mask(s1)
    assert masked_dat.shape()[1]==np.sum(s2.data!=0)

    # Test extract_roi
    mask = create_sphere([41, 64, 55], radius=10)
    assert len(dat.extract_roi(mask))==shape_2d[0]

    # Test r_to_z
    z = dat.r_to_z()
    assert z.shape() == dat.shape()

    # Test copy
    d_copy = dat.copy()
    assert d_copy.shape() == dat.shape()

    # Test detrend
    detrend = dat.detrend()
    assert detrend.shape() == dat.shape()

    # Test Sum
    s = dat.sum()
    assert s.shape()==dat[1].shape()

    # Test Groupby
    s1 = create_sphere([45, 55, 45], radius=10)
    s2 = create_sphere([30, 30, 40], radius=10)
    mask = Brain_Data([s1,s2])
    d = dat.groupby(mask)
    assert isinstance(d,Groupby)

    # Test Aggregate
    mn = dat.aggregate(mask,'mean')
    assert isinstance(mn,Brain_Data)
    assert len(mn.shape())==1

    # Test Threshold
    s1 = create_sphere([45, 55, 45], radius=10)
    s2 = create_sphere([30, 30, 40], radius=10)
    mask = Brain_Data(s1)*5
    mask = mask + Brain_Data(s2)

    m1 = mask.threshold(threshold=.5)
    m2 = mask.threshold(threshold=3)
    m3 = mask.threshold(threshold='98%')
    assert np.sum(m1.data>0) > np.sum(m2.data>0)
    assert np.sum(m1.data>0) == np.sum(m3.data>0)

    # Test Regions
    r = mask.regions(min_region_size=10)
    m1 = Brain_Data(s1)
    m2 = r[1].threshold(1,binarize=True)
    assert len(r)==2
    diff = m2-m1
    assert np.sum(diff.data)==0

    # # Test Plot
    # dat.plot()

    # Test Bootstrap

    # Test multivariate_similarity

def test_adjacency(tmpdir):
    n=10
    sim = np.random.multivariate_normal([0,0,0,0],[[1, 0.8, 0.1, 0.4],
                                         [0.8, 1, 0.6, 0.1],
                                         [0.1, 0.6, 1, 0.3],
                                         [0.4, 0.1, 0.3, 1]],100)
    data = pairwise_distances(sim.T,metric='correlation')
    dat_all = []
    for t in range(n):
        tmp = data
        dat_all.append(tmp)

    dat_single = Adjacency(dat_all[0])
    dat_multiple = Adjacency(dat_all)

    # Test automatic distance/similarity detection
    assert dat_single.matrix_type is 'distance'
    dat_single2 = Adjacency(1-data)
    assert dat_single2.matrix_type is 'similarity'

    # Test length
    assert len(dat_multiple)==dat_multiple.data.shape[0]
    assert len(dat_multiple[0])==1

    # Test Indexing
    assert len(dat_multiple[0]) == 1
    assert len(dat_multiple[0:4]) == 4
    assert len(dat_multiple[0,2,3]) == 3

    # Test copy
    assert np.all(dat_multiple.data==dat_multiple.copy().data)

    # Test squareform & iterable
    assert len(dat_multiple.squareform())==len(dat_multiple)
    assert dat_single.squareform().shape==data.shape

    # Test write
    dat_multiple.write(os.path.join(str(tmpdir.join('Test.csv'))),method='long')
    dat_multiple2 = Adjacency(os.path.join(str(tmpdir.join('Test.csv'))),matrix_type='distance_flat')
    assert np.all(np.isclose(dat_multiple.data,dat_multiple2.data))

    # Test mean
    assert len(dat_multiple.mean(axis=0))==len(np.mean(dat_multiple.data,axis=0))
    assert len(dat_multiple.mean(axis=1))==len(np.mean(dat_multiple.data,axis=1))

    # Test std
    assert len(dat_multiple.std(axis=1))==len(np.std(dat_multiple.data,axis=1))
    assert len(dat_multiple.std(axis=0))==len(np.std(dat_multiple.data,axis=0))

    # Test similarity
    assert len(dat_multiple.similarity(dat_single.squareform()))==len(dat_multiple)

    # Test ttest
    mn,p = dat_multiple.ttest()
    assert len(mn)==1
    assert len(p)==1
    assert mn.shape()[0]==dat_multiple.shape()[1]
    assert p.shape()[0]==dat_multiple.shape()[1]

    # # Test stats_label_distance - FAILED - Need to sort this out
    # labels = np.array(['group1','group1','group2','group2'])
    # stats = dat_multiple[0].stats_label_distance(labels)
    # assert np.isclose(stats['group1']['mean'],-1*stats['group2']['mean'])

def test_groupby(tmpdir):
    # Simulate Brain Data
    sim = Simulator()
    r = 10
    sigma = 1
    y = [0, 1]
    n_reps = 3
    output_dir = str(tmpdir)
    sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)

    s1 = create_sphere([45, 55, 45], radius=r)
    s2 = create_sphere([30, 30, 40], radius=r)
    mask = Brain_Data([s1,s2])

    y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None,index_col=None).T
    data = Brain_Data(glob.glob(str(tmpdir.join('centered*.nii.gz'))),Y=y)
    data.X = pd.DataFrame({'Intercept':np.ones(len(data.Y)),'X1':np.array(data.Y).flatten()},index=None)

    dat = Groupby(data,mask)

    # Test length
    assert len(dat)==len(mask)

    # Test Index
    assert isinstance(dat[1],Brain_Data)

    # Test apply
    mn = dat.apply('mean')
    assert len(dat)==len(mn)
    assert mn[0].mean() > mn[1].mean()
    assert mn[1].shape()==np.sum(mask[1].data==1)
    reg = dat.apply('regress')
    assert len(dat)==len(mn)
    # r = dict([(x,reg[x]['beta'][1]) for x in reg.iterkeys()])

    # Test combine
    combine_mn = dat.combine(mn)
    assert len(combine_mn.shape())==1

<commit_msg>updated tests<commit_after>import os
import numpy as np
import nibabel as nb
import pandas as pd
import glob
from nltools.simulator import Simulator
from nltools.data import Brain_Data, Adjacency, Groupby
from nltools.data import threshold
from nltools.mask import create_sphere
from sklearn.metrics import pairwise_distances
import matplotlib
matplotlib.use('TkAgg')

def test_brain_data(tmpdir):
    sim = Simulator()
    r = 10
    sigma = 1
    y = [0, 1]
    n_reps = 3
    output_dir = str(tmpdir)
    sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)

    shape_3d = (91, 109, 91)
    shape_2d = (6, 238955)
    y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None,index_col=None).T
    flist = glob.glob(str(tmpdir.join('centered*.nii.gz')))

    # Test load list
    dat = Brain_Data(data=flist,Y=y)

    # Test load file
    assert Brain_Data(flist[0])

    # Test to_nifti
    d = dat.to_nifti()
    assert d.shape[0:3] == shape_3d

    # Test load nibabel
    assert Brain_Data(d)

    # Test shape
    assert dat.shape() == shape_2d

    # Test Mean
    assert dat.mean().shape()[0] == shape_2d[1]

    # Test Std
    assert dat.std().shape()[0] == shape_2d[1]

    # Test add
    new = dat + dat
    assert new.shape() == shape_2d

    # Test subtract
    new = dat - dat
    assert new.shape() == shape_2d

    # Test multiply
    new = dat * dat
    assert new.shape() == shape_2d

    # Test Iterator
    x = [x for x in dat]
    assert len(x)==len(dat)
    assert len(x[0].data.shape)==1

    # # Test T-test
    out = dat.ttest()
    assert out['t'].shape()[0]==shape_2d[1]

    # # # Test T-test - permutation method
    # out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1})
    # assert out['t'].shape()[0]==shape_2d[1]

    # Test Regress
    dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)),'X1':np.array(dat.Y).flatten()},index=None)
    out = dat.regress()
    assert out['beta'].shape() == (2,shape_2d[1])

    # Test indexing
    assert out['t'][1].shape()[0] == shape_2d[1]

    # Test threshold
    i=1
    tt = threshold(out['t'][i], out['p'][i], .05)
    assert isinstance(tt,Brain_Data)

    # Test write
    dat.write(os.path.join(str(tmpdir.join('test_write.nii'))))
    assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii'))))

    # Test append
    assert dat.append(dat).shape()[0]==shape_2d[0]*2

    # Test distance
    distance = dat.distance(method='euclidean')
    assert isinstance(distance,Adjacency)
    assert distance.square_shape()[0]==shape_2d[0]

    # Test predict
    stats = dat.predict(algorithm='svm', cv_dict={'type': 'kfolds','n_folds': 2, 'n':len(dat.Y)}, plot=False,**{'kernel':"linear"})

    # Support Vector Regression, with 5 fold cross-validation with Platt Scaling
    # This will output probabilities of each class
    stats = dat.predict(algorithm='svm', cv_dict=None, plot=False,**{'kernel':'linear','probability':True})
    assert isinstance(stats['weight_map'],Brain_Data)

    # Logistic classificiation, with 5 fold stratified cross-validation.
    stats = dat.predict(algorithm='logistic', cv_dict={'type': 'kfolds','n_folds': 5, 'n':len(dat.Y)}, plot=False)
    assert isinstance(stats['weight_map'],Brain_Data)

    # Ridge classificiation, with 5 fold between-subject cross-validation, where data for each subject is held out together.
    stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None,plot=False)
    assert isinstance(stats['weight_map'],Brain_Data)

    # Test Similarity
    r = dat.similarity(stats['weight_map'])
    assert len(r)==shape_2d[0]
    r2 = dat.similarity(stats['weight_map'].to_nifti())
    assert len(r2)==shape_2d[0]

    # Test apply_mask - might move part of this to test mask suite
    s1 = create_sphere([41, 64, 55], radius=10)
    assert isinstance(s1,nb.Nifti1Image)
    s2 = Brain_Data(s1)
    masked_dat = dat.apply_mask(s1)
    assert masked_dat.shape()[1]==np.sum(s2.data!=0)

    # Test extract_roi
    mask = create_sphere([41, 64, 55], radius=10)
    assert len(dat.extract_roi(mask))==shape_2d[0]

    # Test r_to_z
    z = dat.r_to_z()
    assert z.shape() == dat.shape()

    # Test copy
    d_copy = dat.copy()
    assert d_copy.shape() == dat.shape()

    # Test detrend
    detrend = dat.detrend()
    assert detrend.shape() == dat.shape()

    # Test Sum
    s = dat.sum()
    assert s.shape()==dat[1].shape()

    # Test Groupby
    s1 = create_sphere([45, 55, 45], radius=10)
    s2 = create_sphere([30, 30, 40], radius=10)
    mask = Brain_Data([s1,s2])
    d = dat.groupby(mask)
    assert isinstance(d,Groupby)

    # Test Aggregate
    mn = dat.aggregate(mask,'mean')
    assert isinstance(mn,Brain_Data)
    assert len(mn.shape())==1

    # Test Threshold
    s1 = create_sphere([45, 55, 45], radius=10)
    s2 = create_sphere([30, 30, 40], radius=10)
    mask = Brain_Data(s1)*5
    mask = mask + Brain_Data(s2)

    m1 = mask.threshold(threshold=.5)
    m2 = mask.threshold(threshold=3)
    m3 = mask.threshold(threshold='98%')
    assert np.sum(m1.data>0) > np.sum(m2.data>0)
    assert np.sum(m1.data>0) == np.sum(m3.data>0)

    # Test Regions
    r = mask.regions(min_region_size=10)
    m1 = Brain_Data(s1)
    m2 = r[1].threshold(1,binarize=True)
    assert len(r)==2
    diff = m2-m1
    assert np.sum(diff.data)==0

    # # Test Plot
    # dat.plot()

    # Test Bootstrap

    # Test multivariate_similarity

def test_adjacency(tmpdir):
    n=10
    sim = np.random.multivariate_normal([0,0,0,0],[[1, 0.8, 0.1, 0.4],
                                         [0.8, 1, 0.6, 0.1],
                                         [0.1, 0.6, 1, 0.3],
                                         [0.4, 0.1, 0.3, 1]],100)
    data = pairwise_distances(sim.T,metric='correlation')
    dat_all = []
    for t in range(n):
        tmp = data
        dat_all.append(tmp)

    dat_single = Adjacency(dat_all[0])
    dat_multiple = Adjacency(dat_all)

    # Test automatic distance/similarity detection
    assert dat_single.matrix_type is 'distance'
    dat_single2 = Adjacency(1-data)
    assert dat_single2.matrix_type is 'similarity'

    # Test length
    assert len(dat_multiple)==dat_multiple.data.shape[0]
    assert len(dat_multiple[0])==1

    # Test Indexing
    assert len(dat_multiple[0]) == 1
    assert len(dat_multiple[0:4]) == 4
    assert len(dat_multiple[0,2,3]) == 3

    # Test copy
    assert np.all(dat_multiple.data==dat_multiple.copy().data)

    # Test squareform & iterable
    assert len(dat_multiple.squareform())==len(dat_multiple)
    assert dat_single.squareform().shape==data.shape

    # Test write
    dat_multiple.write(os.path.join(str(tmpdir.join('Test.csv'))),method='long')
    dat_multiple2 = Adjacency(os.path.join(str(tmpdir.join('Test.csv'))),matrix_type='distance_flat')
    assert np.all(np.isclose(dat_multiple.data,dat_multiple2.data))

    # Test mean
    assert len(dat_multiple.mean(axis=0))==len(np.mean(dat_multiple.data,axis=0))
    assert len(dat_multiple.mean(axis=1))==len(np.mean(dat_multiple.data,axis=1))

    # Test std
    assert len(dat_multiple.std(axis=1))==len(np.std(dat_multiple.data,axis=1))
    assert len(dat_multiple.std(axis=0))==len(np.std(dat_multiple.data,axis=0))

    # Test similarity
    assert len(dat_multiple.similarity(dat_single.squareform()))==len(dat_multiple)

    # Test ttest
    mn,p = dat_multiple.ttest()
    assert len(mn)==1
    assert len(p)==1
    assert mn.shape()[0]==dat_multiple.shape()[1]
    assert p.shape()[0]==dat_multiple.shape()[1]

    # # Test stats_label_distance - FAILED - Need to sort this out
    # labels = np.array(['group1','group1','group2','group2'])
    # stats = dat_multiple[0].stats_label_distance(labels)
    # assert np.isclose(stats['group1']['mean'],-1*stats['group2']['mean'])

def test_groupby(tmpdir):
    # Simulate Brain Data
    sim = Simulator()
    r = 10
    sigma = 1
    y = [0, 1]
    n_reps = 3
    output_dir = str(tmpdir)
    sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)

    s1 = create_sphere([45, 55, 45], radius=r)
    s2 = create_sphere([30, 30, 40], radius=r)
    mask = Brain_Data([s1,s2])

    y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None,index_col=None).T
    data = Brain_Data(glob.glob(str(tmpdir.join('centered*.nii.gz'))),Y=y)
    data.X = pd.DataFrame({'Intercept':np.ones(len(data.Y)),'X1':np.array(data.Y).flatten()},index=None)

    dat = Groupby(data,mask)

    # Test length
    assert len(dat)==len(mask)

    # Test Index
    assert isinstance(dat[1],Brain_Data)

    # Test apply
    mn = dat.apply('mean')
    assert len(dat)==len(mn)
    assert mn[0].mean() > mn[1].mean()
    assert mn[1].shape()==np.sum(mask[1].data==1)
    reg = dat.apply('regress')
    assert len(dat)==len(mn)
    # r = dict([(x,reg[x]['beta'][1]) for x in reg.iterkeys()])

    # Test combine
    combine_mn = dat.combine(mn)
    assert len(combine_mn.shape())==1

<|endoftext|>