<commit_before>#!/usr/bin/env python

import networkx as nx
import subprocess as sp
from eden.converter.fasta import seq_to_networkx
from eden.converter.rna import sequence_dotbracket_to_graph
from eden.util import is_iterable

# TODO: use sampling if seq is too long
# use different parsing of output in this case
# check version of program

def rnashapes_wrapper(sequence, shape_type=None, energy_range=None, max_num=None):
    # command line
    cmd = 'echo "%s" | RNAshapes -t %d -c %d -# %d' % (sequence, shape_type, energy_range, max_num)
    out = sp.check_output(cmd, shell=True)
    # parse output
    text = out.strip().split('\n')
    seq_info = text[0]
    if 'configured to print' in text[-1]:
        struct_text = text[1:-1]
    else:
        struct_text = text[1:]
    seq_struct_list = [line.split()[1] for line in struct_text]
    struct_list = [line.split()[2] for line in struct_text]
    return seq_info, seq_struct_list, struct_list

def string_to_networkx(header, sequence, **options):
    # defaults
    shape_type = options.get('shape_type', 5)
    energy_range = options.get('energy_range', 10)
    max_num = options.get('max_num', 3)
    split_components = options.get('split_components', False)
    seq_info, seq_struct_list, struct_list = rnashapes_wrapper(sequence,
                                                               shape_type=shape_type,
                                                               energy_range=energy_range,
                                                               max_num=max_num)
    if split_components:
        for seq_struct, struct in zip(seq_struct_list, struct_list):
            graph = sequence_dotbracket_to_graph(seq_info=seq_info, seq_struct=seq_struct)
            graph.graph['info'] = 'RNAshapes shape_type=%s energy_range=%s max_num=%s' % (shape_type,
                                                                                energy_range,
                                                                                max_num)
            graph.graph['id'] = header + '_' + struct
            if graph.number_of_nodes() < 2:
                graph = seq_to_networkx(header, sequence, **options)
                graph.graph['id'] = header
            graph.graph['sequence'] = sequence
            graph.graph['structure'] = seq_struct
            yield graph
    else:
        graph_global = nx.Graph()
        graph_global.graph['id'] = header
        graph_global.graph['info'] = 'RNAshapes shape_type=%s energy_range=%s max_num=%s' % (shape_type,
                                                                                energy_range,
                                                                                max_num)
        graph_global.graph['sequence'] = sequence
        for seq_struct in seq_struct_list:
            graph = sequence_dotbracket_to_graph(seq_info=seq_info, seq_struct=seq_struct)
            graph_global = nx.disjoint_union(graph_global, graph)
        if graph_global.number_of_nodes() < 2:
            graph_global = seq_to_networkx(header, sequence, **options)
        yield graph_global

def rnashapes_to_eden(iterable, **options):
    """Transforms sequences to graphs that encode secondary structure information
    according to the RNAShapes algorithm.

    Parameters
    ----------
    sequences : iterable
        iterable pairs of header and sequence strings

    shape_type : int (default 5)
        Is the level of abstraction or dissimilarity which defines a different shape.
        In general, helical regions are depicted by a pair of opening and closing brackets
        and unpaired regions are represented as a single underscore. The differences of the
        shape types are due to whether a structural element (bulge loop, internal loop, multiloop,
        hairpin loop, stacking region and external loop) contributes to the shape representation:
        Five types are implemented.
        1   Most accurate - all loops and all unpaired  [_[_[]]_[_[]_]]_
        2   Nesting pattern for all loop types and unpaired regions in external loop
        and multiloop [[_[]][_[]_]]
        3   Nesting pattern for all loop types but no unpaired regions [[[]][[]]]
        4   Helix nesting pattern in external loop and multiloop [[][[]]]
        5   Most abstract - helix nesting pattern and no unpaired regions [[][]]

    energy_range : float (default 10)
        Sets the energy range as percentage value of the minimum free energy.
        For example, when relative deviation is specified as 5.0, and the minimum free energy
        is -10.0 kcal/mol, the energy range is set to -9.5 to -10.0 kcal/mol.
        Relative deviation must be a positive floating point number; by default it is set to to 10 %.

    max_num : int (default 3)
        Is the maximum number of structures that are generated.

    split_components : bool (default False)
        If True each structure is yielded as an independent graph. Otherwise all structures
        are part of the same graph that has therefore several disconnectd components.

    >>> # transform a simple sequence
    >>> graphs = rnashapes_to_eden([("ID", "CCCCCGGGGG")])
    >>> g = graphs.next()
    >>> # extract sequence from graph nodes
    >>> "".join([ value["label"] for (key, value) in g.nodes(data=True)])
    'CCCCCGGGGG'
    >>> # get vertice types
    >>> [ (start, end, g.edge[start][end]["type"]) for start, end in g.edges()]
    [(0, 1, 'backbone'), (0, 9, 'basepair'), (1, 8, 'basepair'), (1, 2, 'backbone'), (2, 3, 'backbone'), (2, 7, 'basepair'), (3, 4, 'backbone'), (4, 5, 'backbone'), (5, 6, 'backbone'), (6, 7, 'backbone'), (7, 8, 'backbone'), (8, 9, 'backbone')]
    """

    assert(is_iterable(iterable)), 'Not iterable'
    for header, seq in iterable:
        try:
            for graph in string_to_networkx(header, seq, **options):
                yield graph
        except Exception as e:
            print e.__doc__
            print e.message
            print 'Error in: %s' % seq
            graph = seq_to_networkx(header, seq, **options)
            yield graph
<commit_msg>add doctest for rnashapes<commit_after>#!/usr/bin/env python

import networkx as nx
import subprocess as sp
from eden.converter.fasta import seq_to_networkx
from eden.converter.rna import sequence_dotbracket_to_graph
from eden.util import is_iterable

# TODO: use sampling if seq is too long
# use different parsing of output in this case
# check version of program

def rnashapes_wrapper(sequence, shape_type=None, energy_range=None, max_num=None):
    # command line
    cmd = 'echo "%s" | RNAshapes -t %d -c %d -# %d' % (sequence, shape_type, energy_range, max_num)
    out = sp.check_output(cmd, shell=True)
    # parse output
    text = out.strip().split('\n')
    seq_info = text[0]
    if 'configured to print' in text[-1]:
        struct_text = text[1:-1]
    else:
        struct_text = text[1:]
    seq_struct_list = [line.split()[1] for line in struct_text]
    struct_list = [line.split()[2] for line in struct_text]
    return seq_info, seq_struct_list, struct_list

def string_to_networkx(header, sequence, **options):
    # defaults
    shape_type = options.get('shape_type', 5)
    energy_range = options.get('energy_range', 10)
    max_num = options.get('max_num', 3)
    split_components = options.get('split_components', False)
    seq_info, seq_struct_list, struct_list = rnashapes_wrapper(sequence,
                                                               shape_type=shape_type,
                                                               energy_range=energy_range,
                                                               max_num=max_num)
    if split_components:
        for seq_struct, struct in zip(seq_struct_list, struct_list):
            graph = sequence_dotbracket_to_graph(seq_info=seq_info, seq_struct=seq_struct)
            graph.graph['info'] = 'RNAshapes shape_type=%s energy_range=%s max_num=%s' % (shape_type,
                                                                                energy_range,
                                                                                max_num)
            graph.graph['id'] = header + '_' + struct
            if graph.number_of_nodes() < 2:
                graph = seq_to_networkx(header, sequence, **options)
                graph.graph['id'] = header
            graph.graph['sequence'] = sequence
            graph.graph['structure'] = seq_struct
            yield graph
    else:
        graph_global = nx.Graph()
        graph_global.graph['id'] = header
        graph_global.graph['info'] = 'RNAshapes shape_type=%s energy_range=%s max_num=%s' % (shape_type,
                                                                                energy_range,
                                                                                max_num)
        graph_global.graph['sequence'] = sequence
        for seq_struct in seq_struct_list:
            graph = sequence_dotbracket_to_graph(seq_info=seq_info, seq_struct=seq_struct)
            graph_global = nx.disjoint_union(graph_global, graph)
        if graph_global.number_of_nodes() < 2:
            graph_global = seq_to_networkx(header, sequence, **options)
        yield graph_global

def rnashapes_to_eden(iterable, **options):
    """Transforms sequences to graphs that encode secondary structure information
    according to the RNAShapes algorithm.

    Parameters
    ----------
    sequences : iterable
        iterable pairs of header and sequence strings

    shape_type : int (default 5)
        Is the level of abstraction or dissimilarity which defines a different shape.
        In general, helical regions are depicted by a pair of opening and closing brackets
        and unpaired regions are represented as a single underscore. The differences of the
        shape types are due to whether a structural element (bulge loop, internal loop, multiloop,
        hairpin loop, stacking region and external loop) contributes to the shape representation:
        Five types are implemented.
        1   Most accurate - all loops and all unpaired  [_[_[]]_[_[]_]]_
        2   Nesting pattern for all loop types and unpaired regions in external loop
        and multiloop [[_[]][_[]_]]
        3   Nesting pattern for all loop types but no unpaired regions [[[]][[]]]
        4   Helix nesting pattern in external loop and multiloop [[][[]]]
        5   Most abstract - helix nesting pattern and no unpaired regions [[][]]

    energy_range : float (default 10)
        Sets the energy range as percentage value of the minimum free energy.
        For example, when relative deviation is specified as 5.0, and the minimum free energy
        is -10.0 kcal/mol, the energy range is set to -9.5 to -10.0 kcal/mol.
        Relative deviation must be a positive floating point number; by default it is set to to 10 %.

    max_num : int (default 3)
        Is the maximum number of structures that are generated.

    split_components : bool (default False)
        If True each structure is yielded as an independent graph. Otherwise all structures
        are part of the same graph that has therefore several disconnectd components.

    >>> # transform a simple sequence
    >>> graphs = rnashapes_to_eden([("ID", "CCCCCGGGGG")])
    >>> g = graphs.next()
    >>> # extract sequence from graph nodes
    >>> "".join([ value["label"] for (key, value) in g.nodes(data=True)])
    'CCCCCGGGGG'
    >>> # get vertice types
    >>> [ (start, end, g.edge[start][end]["type"]) for start, end in g.edges()]
    [(0, 1, 'backbone'), (0, 9, 'basepair'), (1, 8, 'basepair'), (1, 2, 'backbone'), (2, 3, 'backbone'), (2, 7, 'basepair'), (3, 4, 'backbone'), (4, 5, 'backbone'), (5, 6, 'backbone'), (6, 7, 'backbone'), (7, 8, 'backbone'), (8, 9, 'backbone')]

    >>> # transform a simple sequence, splitting components
    >>> graphs = rnashapes_to_eden([("ID", "CCCCCGGGGG")], split_components=True)
    >>> g = graphs.next()
    >>> # extract sequence from graph nodes
    >>> "".join([ value["label"] for (key, value) in g.nodes(data=True)])
    'CCCCCGGGGG'
    >>> # get dotbracket structure annotation
    >>> g.graph["structure"]
    '(((....)))'
    >>> # get vertice types
    >>> [ (start, end, g.edge[start][end]["type"]) for start, end in g.edges()]
    [(0, 1, 'backbone'), (0, 9, 'basepair'), (1, 8, 'basepair'), (1, 2, 'backbone'), (2, 3, 'backbone'), (2, 7, 'basepair'), (3, 4, 'backbone'), (4, 5, 'backbone'), (5, 6, 'backbone'), (6, 7, 'backbone'), (7, 8, 'backbone'), (8, 9, 'backbone')]
    """

    assert(is_iterable(iterable)), 'Not iterable'
    for header, seq in iterable:
        try:
            for graph in string_to_networkx(header, seq, **options):
                yield graph
        except Exception as e:
            print e.__doc__
            print e.message
            print 'Error in: %s' % seq
            graph = seq_to_networkx(header, seq, **options)
            yield graph
<|endoftext|>


<commit_before>from semantix.utils.json import _encoder
from semantix.utils.json._encoder import Encoder
import sys
from decimal import Decimal
from uuid import UUID

exec(''.join(sys.argv[1:]), locals(), globals())

<commit_msg>utils.json: Remove debug script<commit_after><|endoftext|>


<commit_before>from __future__ import unicode_literals

import random

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import FormView

from candidates.views.mixins import ContributorsMixin
from candidates.models import PostExtra, PostExtraElection
from tasks.models import PersonTask

from elections.models import Election

from ..forms import PostcodeForm
from ..mapit import get_areas_from_postcode

class ConstituencyPostcodeFinderView(ContributorsMixin, FormView):
    template_name = 'candidates/finder.html'
    form_class = PostcodeForm

    @method_decorator(cache_control(max_age=(60 * 10)))
    @method_decorator(csrf_exempt)
    def dispatch(self, *args, **kwargs):
        return super(ConstituencyPostcodeFinderView, self).dispatch(*args, **kwargs)

    def process_postcode(self, postcode):
        types_and_areas = get_areas_from_postcode(postcode)
        if settings.AREAS_TO_ALWAYS_RETURN:
            types_and_areas += settings.AREAS_TO_ALWAYS_RETURN
        types_and_areas_joined = ','.join(sorted(
            '{0}--{1}'.format(*t) for t in types_and_areas
        ))
        return HttpResponseRedirect(
            reverse('areas-view', kwargs={
                'type_and_area_ids': types_and_areas_joined
            })
        )

    def get_form_kwargs(self):
        if self.request.method == 'GET' and 'q' in self.request.GET:
            return {
                'data': self.request.GET,
                'initial': self.get_initial(),
                'prefix': self.get_prefix(),
            }
        else:
            return super(ConstituencyPostcodeFinderView, self).get_form_kwargs()

    def get(self, request, *args, **kwargs):
        if 'q' in request.GET:
            # The treat it like a POST request; we've overridden
            # get_form_kwargs to make sure the GET parameters are used
            # for the form in this case.
            return self.post(request, *args, **kwargs)
        else:
            return super(ConstituencyPostcodeFinderView, self).get(request, *args, **kwargs)

    def form_valid(self, form):
        return self.process_postcode(form.cleaned_data['q'])

    def sopn_progress_by_election_slug_prefix(self, election_slug_prefix):
        election_qs = Election.objects.filter(slug__startswith=election_slug_prefix)
        return self.sopn_progress_by_election(election_qs)

    def sopn_progress_by_election(self, election_qs):
        context = {}
        if not election_qs.exists():
            return context
        pee_qs = PostExtraElection.objects.filter(election__in=election_qs)

        context['posts_total'] = pee_qs.count()
        pee_qs = pee_qs.filter(candidates_locked=True, election__in=election_qs)
        context['posts_locked'] = pee_qs.count()
        context['posts_locked_percent'] = round(
                float(context['posts_locked']) /
                float(context['posts_total'])
                * 100)

        context['posts_lock_suggested'] = pee_qs.exclude(
            suggestedpostlock=None).count()
        context['posts_locked_suggested_percent'] = round(
                float(context['posts_lock_suggested']) /
                float(context['posts_total'])
                * 100)

        context['sopns_imported'] = pee_qs.exclude(
            postextra__base__officialdocument=None).count()
        context['sopns_imported_percent'] = round(
                float(context['sopns_imported']) /
                float(context['posts_total'])
                * 100)

        return context

    def get_context_data(self, **kwargs):
        context = super(ConstituencyPostcodeFinderView, self).get_context_data(**kwargs)
        context['postcode_form'] = kwargs.get('form') or PostcodeForm()
        context['show_postcode_form'] = True
        context['show_name_form'] = False
        context['top_users'] = self.get_leaderboards(all_time=False)[0]['rows'][:8]
        context['recent_actions'] = self.get_recent_changes_queryset()[:5]
        context['election_data'] = Election.objects.current().by_date().last()
        context['hide_search_form'] = True

        from uk_results.models import CouncilElection
        context['council_total'] = CouncilElection.objects.all().count()
        context['council_confirmed'] = CouncilElection.objects.filter(
            confirmed=True).count()

        if context['council_total']:
            context['council_election_percent'] = round(
                float(context['council_confirmed']) /
                float(context['council_total'])
                * 100)
        else:
            context['council_election_percent'] = 0

        from candidates.models import PostExtra
        from uk_results.models import PostElectionResult
        context['votes_total'] = PostExtra.objects.filter(
            postextraelection__election__slug__contains="local").count()
        context['votes_confirmed'] = PostElectionResult.objects.filter(
            confirmed=True).count()

        if float(context['votes_confirmed']):
            context['votes_percent'] = round(
                float(context['votes_confirmed']) /
                float(context['votes_total'])
                * 100)
        else:
            context['votes_percent'] = 0

        # context['council_election_percent'] = council_confirmed / council_total * 100

        election_qs = Election.objects.filter(
            election_date="2017-06-08")
        context['sopn_progress'] = self.sopn_progress_by_election(
            election_qs=election_qs)

        task_count = PersonTask.objects.unfinished_tasks().count()
        if task_count > 0:
            random_offset = random.randrange(min(50, task_count))
            context['person_task'] = PersonTask.objects.unfinished_tasks()[random_offset]

        return context
<commit_msg>Fix SOPN imported count<commit_after>from __future__ import unicode_literals

import random

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import FormView
from django.db.models import F

from candidates.views.mixins import ContributorsMixin
from candidates.models import PostExtra, PostExtraElection
from tasks.models import PersonTask

from elections.models import Election

from ..forms import PostcodeForm
from ..mapit import get_areas_from_postcode

class ConstituencyPostcodeFinderView(ContributorsMixin, FormView):
    template_name = 'candidates/finder.html'
    form_class = PostcodeForm

    @method_decorator(cache_control(max_age=(60 * 10)))
    @method_decorator(csrf_exempt)
    def dispatch(self, *args, **kwargs):
        return super(ConstituencyPostcodeFinderView, self).dispatch(*args, **kwargs)

    def process_postcode(self, postcode):
        types_and_areas = get_areas_from_postcode(postcode)
        if settings.AREAS_TO_ALWAYS_RETURN:
            types_and_areas += settings.AREAS_TO_ALWAYS_RETURN
        types_and_areas_joined = ','.join(sorted(
            '{0}--{1}'.format(*t) for t in types_and_areas
        ))
        return HttpResponseRedirect(
            reverse('areas-view', kwargs={
                'type_and_area_ids': types_and_areas_joined
            })
        )

    def get_form_kwargs(self):
        if self.request.method == 'GET' and 'q' in self.request.GET:
            return {
                'data': self.request.GET,
                'initial': self.get_initial(),
                'prefix': self.get_prefix(),
            }
        else:
            return super(ConstituencyPostcodeFinderView, self).get_form_kwargs()

    def get(self, request, *args, **kwargs):
        if 'q' in request.GET:
            # The treat it like a POST request; we've overridden
            # get_form_kwargs to make sure the GET parameters are used
            # for the form in this case.
            return self.post(request, *args, **kwargs)
        else:
            return super(ConstituencyPostcodeFinderView, self).get(request, *args, **kwargs)

    def form_valid(self, form):
        return self.process_postcode(form.cleaned_data['q'])

    def sopn_progress_by_election_slug_prefix(self, election_slug_prefix):
        election_qs = Election.objects.filter(slug__startswith=election_slug_prefix)
        return self.sopn_progress_by_election(election_qs)

    def sopn_progress_by_election(self, election_qs):
        context = {}
        if not election_qs.exists():
            return context
        pee_qs = PostExtraElection.objects.filter(election__in=election_qs)

        context['sopns_imported'] = pee_qs.filter(
                postextra__base__officialdocument__election=F('election'),
            ).count()

        context['posts_total'] = pee_qs.count()
        pee_qs = pee_qs.filter(candidates_locked=True, election__in=election_qs)
        context['posts_locked'] = pee_qs.count()
        context['posts_locked_percent'] = round(
                float(context['posts_locked']) /
                float(context['posts_total'])
                * 100)

        context['posts_lock_suggested'] = pee_qs.exclude(
            suggestedpostlock=None).count()
        context['posts_locked_suggested_percent'] = round(
                float(context['posts_lock_suggested']) /
                float(context['posts_total'])
                * 100)

        context['sopns_imported_percent'] = round(
                float(context['sopns_imported']) /
                float(context['posts_total'])
                * 100)

        return context

    def get_context_data(self, **kwargs):
        context = super(ConstituencyPostcodeFinderView, self).get_context_data(**kwargs)
        context['postcode_form'] = kwargs.get('form') or PostcodeForm()
        context['show_postcode_form'] = True
        context['show_name_form'] = False
        context['top_users'] = self.get_leaderboards(all_time=False)[0]['rows'][:8]
        context['recent_actions'] = self.get_recent_changes_queryset()[:5]
        context['election_data'] = Election.objects.current().by_date().last()
        context['hide_search_form'] = True

        from uk_results.models import CouncilElection
        context['council_total'] = CouncilElection.objects.all().count()
        context['council_confirmed'] = CouncilElection.objects.filter(
            confirmed=True).count()

        if context['council_total']:
            context['council_election_percent'] = round(
                float(context['council_confirmed']) /
                float(context['council_total'])
                * 100)
        else:
            context['council_election_percent'] = 0

        from candidates.models import PostExtra
        from uk_results.models import PostElectionResult
        context['votes_total'] = PostExtra.objects.filter(
            postextraelection__election__slug__contains="local").count()
        context['votes_confirmed'] = PostElectionResult.objects.filter(
            confirmed=True).count()

        if float(context['votes_confirmed']):
            context['votes_percent'] = round(
                float(context['votes_confirmed']) /
                float(context['votes_total'])
                * 100)
        else:
            context['votes_percent'] = 0

        # context['council_election_percent'] = council_confirmed / council_total * 100

        election_qs = Election.objects.filter(
            election_date="2017-06-08")
        context['sopn_progress'] = self.sopn_progress_by_election(
            election_qs=election_qs)

        task_count = PersonTask.objects.unfinished_tasks().count()
        if task_count > 0:
            random_offset = random.randrange(min(50, task_count))
            context['person_task'] = PersonTask.objects.unfinished_tasks()[random_offset]

        return context
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from singledispatch import singledispatch

import numpy as np  # type: ignore

from sklearn.base import BaseEstimator  # type: ignore
from sklearn.linear_model import (   # type: ignore
    ElasticNet,  # includes Lasso, MultiTaskElasticNet, etc.
    ElasticNetCV,
    HuberRegressor,
    Lars,
    LassoCV,
    LinearRegression,
    LogisticRegression,
    LogisticRegressionCV,
    OrthogonalMatchingPursuit,
    OrthogonalMatchingPursuitCV,
    PassiveAggressiveClassifier,
    PassiveAggressiveRegressor,
    Perceptron,
    Ridge,
    RidgeClassifier,
    RidgeClassifierCV,
    RidgeCV,
    SGDClassifier,
    SGDRegressor,
    TheilSenRegressor,
)
from sklearn.multiclass import OneVsRestClassifier  # type: ignore
from sklearn.svm import LinearSVC, LinearSVR  # type: ignore
# TODO: see https://github.com/scikit-learn/scikit-learn/pull/2250
from sklearn.naive_bayes import BernoulliNB, MultinomialNB    # type: ignore
from sklearn.ensemble import (  # type: ignore
    GradientBoostingClassifier,
    GradientBoostingRegressor,
    AdaBoostClassifier,
    AdaBoostRegressor,
    RandomForestClassifier,
    RandomForestRegressor,
    ExtraTreesClassifier,
    ExtraTreesRegressor,
)
from sklearn.tree import (  # type: ignore
    DecisionTreeClassifier,
    DecisionTreeRegressor,
)

from eli5.base import (
    Explanation, TargetExplanation, FeatureWeight, FeatureImportances)
from eli5._feature_weights import get_top_features
from eli5.utils import argsort_k_largest_positive, get_target_display_names
from eli5.sklearn.unhashing import handle_hashing_vec, is_invhashing
from eli5.sklearn.treeinspect import get_tree_info
from eli5.sklearn.utils import (
    get_coef,
    is_multiclass_classifier,
    is_multitarget_regressor,
    get_feature_names,
    get_default_target_names,
)
from eli5.explain import explain_weights

LINEAR_CAVEATS = """
Caveats:
1. Be careful with features which are not
   independent - weights don't show their importance.
2. If scale of input features is different then scale of coefficients
   will also be different, making direct comparison between coefficient values
   incorrect.
3. Depending on regularization, rare features sometimes may have high
   coefficients; this doesn't mean they contribute much to the
   classification result for most examples.
""".lstrip()

HASHING_CAVEATS = """
Feature names are restored from their hashes; this is not 100% precise
because collisions are possible. For known collisions possible feature names
are separated by | sign. Keep in mind the collision list is not exhaustive.
Features marked with (-) should be read as inverted: if they have positive
coefficient, the result is negative, if they have negative coefficient,
the result is positive.
""".lstrip()

DESCRIPTION_CLF_MULTICLASS = """
Features with largest coefficients per class.
""" + LINEAR_CAVEATS

DESCRIPTION_CLF_BINARY = """
Features with largest coefficients.
""" + LINEAR_CAVEATS

DESCRIPTION_REGRESSION = DESCRIPTION_CLF_BINARY

DESCRIPTION_REGRESSION_MULTITARGET = """
Features with largest coefficients per target.
""" + LINEAR_CAVEATS

DESCRIPTION_RANDOM_FOREST = """
Random forest feature importances; values are numbers 0 <= x <= 1;
all values sum to 1.
"""

DESCRIPTION_DECISION_TREE = """
Decision tree feature importances; values are numbers 0 <= x <= 1;
all values sum to 1.
"""

_TOP = 20

@explain_weights.register(BaseEstimator)
@singledispatch
def explain_weights_sklearn(estimator, vec=None, top=_TOP,
                            target_names=None,
                            targets=None,
                            feature_names=None, coef_scale=None,
                            feature_re=None):
    """ Return an explanation of an estimator """
    return Explanation(
        estimator=repr(estimator),
        error="estimator %r is not supported" % estimator,
    )

@explain_weights.register(OneVsRestClassifier)
def explain_weights_ovr(ovr, **kwargs):
    estimator = ovr.estimator
    func = explain_weights.dispatch(estimator.__class__)
    return func(ovr, **kwargs)

@explain_weights_sklearn.register(OneVsRestClassifier)
def explain_weights_ovr_sklearn(ovr, **kwargs):
    # dispatch OvR to eli5.sklearn
    # if explain_prediction_sklearn is called explicitly
    estimator = ovr.estimator
    func = explain_weights_sklearn.dispatch(estimator.__class__)
    return func(ovr, **kwargs)

@explain_weights_sklearn.register(LogisticRegression)
@explain_weights_sklearn.register(LogisticRegressionCV)
@explain_weights_sklearn.register(SGDClassifier)
@explain_weights_sklearn.register(PassiveAggressiveClassifier)
@explain_weights_sklearn.register(Perceptron)
@explain_weights_sklearn.register(LinearSVC)
@explain_weights_sklearn.register(RidgeClassifier)
@explain_weights_sklearn.register(RidgeClassifierCV)
def explain_linear_classifier_weights(clf,
                                      vec=None,
                                      top=_TOP,
                                      target_names=None,
                                      targets=None,
                                      feature_names=None,
                                      coef_scale=None,
                                      feature_re=None
                                      ):
    """
    Return an explanation of a linear classifier weights.

    See :func:`eli5.explain_weights` for description of
    ``top``, ``target_names``, ``targets``, ``feature_names`` and
    ``feature_re`` parameters.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the classifier ``clf``
    (e.g. a fitted CountVectorizer instance); you can pass it
    instead of ``feature_names``.

    ``coef_scale`` is a 1D np.ndarray with a scaling coefficient
    for each feature; coef[i] = coef[i] * coef_scale[i] if
    coef_scale[i] is not nan. Use it if you want to scale coefficients
    before displaying them, to take input feature sign or scale in account.
    """
    feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
                                                   coef_scale)
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)

    _extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''

    def _features(label_id):
        coef = get_coef(clf, label_id, scale=coef_scale)
        if feature_re is not None:
            coef = coef[flt_indices]
        return get_top_features(feature_names, coef, top)

    display_names = get_target_display_names(clf.classes_, target_names, targets)
    if is_multiclass_classifier(clf):
        return Explanation(
            targets=[
                TargetExplanation(
                    target=label,
                    feature_weights=_features(label_id)
                )
                for label_id, label in display_names
                ],
            description=DESCRIPTION_CLF_MULTICLASS + _extra_caveats,
            estimator=repr(clf),
            method='linear model',
        )
    else:
        # for binary classifiers scikit-learn stores a single coefficient
        # vector, which corresponds to clf.classes_[1].
        return Explanation(
            targets=[
                TargetExplanation(
                    target=display_names[1][1],
                    feature_weights=_features(0),
                )
            ],
            description=DESCRIPTION_CLF_BINARY + _extra_caveats,
            estimator=repr(clf),
            method='linear model',
        )

@explain_weights_sklearn.register(RandomForestClassifier)
@explain_weights_sklearn.register(RandomForestRegressor)
@explain_weights_sklearn.register(ExtraTreesClassifier)
@explain_weights_sklearn.register(ExtraTreesRegressor)
@explain_weights_sklearn.register(GradientBoostingClassifier)
@explain_weights_sklearn.register(GradientBoostingRegressor)
@explain_weights_sklearn.register(AdaBoostClassifier)
@explain_weights_sklearn.register(AdaBoostRegressor)
def explain_rf_feature_importance(clf,
                                  vec=None,
                                  top=_TOP,
                                  target_names=None,  # ignored
                                  targets=None,       # ignored
                                  feature_names=None,
                                  feature_re=None):
    """
    Return an explanation of a tree-based ensemble estimator.
    """
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    coef = clf.feature_importances_
    trees = np.array(clf.estimators_).ravel()
    coef_std = np.std([tree.feature_importances_ for tree in trees], axis=0)

    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
        coef = coef[flt_indices]
        coef_std = coef_std[flt_indices]

    indices = argsort_k_largest_positive(coef, top)
    names, values, std = feature_names[indices], coef[indices], coef_std[indices]
    return Explanation(
        feature_importances=FeatureImportances(
            [FeatureWeight(*x) for x in zip(names, values, std)],
            remaining=np.count_nonzero(coef) - len(indices),
        ),
        description=DESCRIPTION_RANDOM_FOREST,
        estimator=repr(clf),
        method='feature importances',
    )

@explain_weights_sklearn.register(DecisionTreeClassifier)
@explain_weights_sklearn.register(DecisionTreeRegressor)
def explain_decision_tree(clf,
                          vec=None,
                          top=_TOP,
                          target_names=None,
                          targets=None,  # ignored
                          feature_names=None,
                          feature_re=None,
                          **export_graphviz_kwargs):
    """
    Return an explanation of a decision tree.
    """
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    coef = clf.feature_importances_
    tree_feature_names = feature_names
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
        coef = coef[flt_indices]
    indices = argsort_k_largest_positive(coef, top)
    names, values = feature_names[indices], coef[indices]
    export_graphviz_kwargs.setdefault("proportion", True)
    tree_info = get_tree_info(
        clf,
        feature_names=tree_feature_names,
        class_names=target_names,
        **export_graphviz_kwargs)

    return Explanation(
        feature_importances=FeatureImportances(
            [FeatureWeight(*x) for x in zip(names, values)],
            remaining=np.count_nonzero(coef) - len(indices),
        ),
        decision_tree=tree_info,
        description=DESCRIPTION_DECISION_TREE,
        estimator=repr(clf),
        method='decision tree',
    )

@explain_weights_sklearn.register(ElasticNet)
@explain_weights_sklearn.register(ElasticNetCV)
@explain_weights_sklearn.register(HuberRegressor)
@explain_weights_sklearn.register(Lars)
@explain_weights_sklearn.register(LassoCV)
@explain_weights_sklearn.register(LinearRegression)
@explain_weights_sklearn.register(LinearSVR)
@explain_weights_sklearn.register(OrthogonalMatchingPursuit)
@explain_weights_sklearn.register(OrthogonalMatchingPursuitCV)
@explain_weights_sklearn.register(PassiveAggressiveRegressor)
@explain_weights_sklearn.register(Ridge)
@explain_weights_sklearn.register(RidgeCV)
@explain_weights_sklearn.register(SGDRegressor)
@explain_weights_sklearn.register(TheilSenRegressor)
def explain_linear_regressor_weights(reg,
                                     vec=None,
                                     top=_TOP,
                                     target_names=None,
                                     targets=None,
                                     feature_names=None,
                                     coef_scale=None,
                                     feature_re=None
                                     ):
    """
    Return an explanation of a linear regressor weights.
    See :func:`eli5.explain_weights` for description of
    ``top``, ``target_names``, ``targets``, ``feature_names`` and
    ``feature_re`` parameters.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the regressor ``reg``; you can
    pass it instead of ``feature_names``.

    ``coef_scale`` is a 1D np.ndarray with a scaling coefficient
    for each feature; coef[i] = coef[i] * coef_scale[i] if
    coef_scale[i] is not nan. Use it if you want to scale coefficients
    before displaying them, to take input feature sign or scale in account.
    """
    feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
                                                   coef_scale)
    feature_names = get_feature_names(reg, vec, feature_names=feature_names)
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
    _extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''

    def _features(target_id):
        coef = get_coef(reg, target_id, scale=coef_scale)
        if feature_re is not None:
            coef = coef[flt_indices]
        return get_top_features(feature_names, coef, top)

    display_names = get_target_display_names(get_default_target_names(reg),
                                             target_names, targets)
    if is_multitarget_regressor(reg):
        return Explanation(
            targets=[
                TargetExplanation(
                    target=target_name,
                    feature_weights=_features(target_id)
                )
                for target_id, target_name in display_names
                ],
            description=DESCRIPTION_REGRESSION_MULTITARGET + _extra_caveats,
            estimator=repr(reg),
            method='linear model',
            is_regression=True,
        )
    else:
        return Explanation(
            targets=[TargetExplanation(
                target=display_names[0][1],
                feature_weights=_features(0),
            )],
            description=DESCRIPTION_REGRESSION + _extra_caveats,
            estimator=repr(reg),
            method='linear model',
            is_regression=True,
        )
<commit_msg>DOC better docstrings for explain_decision_tree and explain_rf_feature_importance<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from singledispatch import singledispatch

import numpy as np  # type: ignore

from sklearn.base import BaseEstimator  # type: ignore
from sklearn.linear_model import (   # type: ignore
    ElasticNet,  # includes Lasso, MultiTaskElasticNet, etc.
    ElasticNetCV,
    HuberRegressor,
    Lars,
    LassoCV,
    LinearRegression,
    LogisticRegression,
    LogisticRegressionCV,
    OrthogonalMatchingPursuit,
    OrthogonalMatchingPursuitCV,
    PassiveAggressiveClassifier,
    PassiveAggressiveRegressor,
    Perceptron,
    Ridge,
    RidgeClassifier,
    RidgeClassifierCV,
    RidgeCV,
    SGDClassifier,
    SGDRegressor,
    TheilSenRegressor,
)
from sklearn.multiclass import OneVsRestClassifier  # type: ignore
from sklearn.svm import LinearSVC, LinearSVR  # type: ignore
# TODO: see https://github.com/scikit-learn/scikit-learn/pull/2250
from sklearn.naive_bayes import BernoulliNB, MultinomialNB    # type: ignore
from sklearn.ensemble import (  # type: ignore
    GradientBoostingClassifier,
    GradientBoostingRegressor,
    AdaBoostClassifier,
    AdaBoostRegressor,
    RandomForestClassifier,
    RandomForestRegressor,
    ExtraTreesClassifier,
    ExtraTreesRegressor,
)
from sklearn.tree import (  # type: ignore
    DecisionTreeClassifier,
    DecisionTreeRegressor,
)

from eli5.base import (
    Explanation, TargetExplanation, FeatureWeight, FeatureImportances)
from eli5._feature_weights import get_top_features
from eli5.utils import argsort_k_largest_positive, get_target_display_names
from eli5.sklearn.unhashing import handle_hashing_vec, is_invhashing
from eli5.sklearn.treeinspect import get_tree_info
from eli5.sklearn.utils import (
    get_coef,
    is_multiclass_classifier,
    is_multitarget_regressor,
    get_feature_names,
    get_default_target_names,
)
from eli5.explain import explain_weights

LINEAR_CAVEATS = """
Caveats:
1. Be careful with features which are not
   independent - weights don't show their importance.
2. If scale of input features is different then scale of coefficients
   will also be different, making direct comparison between coefficient values
   incorrect.
3. Depending on regularization, rare features sometimes may have high
   coefficients; this doesn't mean they contribute much to the
   classification result for most examples.
""".lstrip()

HASHING_CAVEATS = """
Feature names are restored from their hashes; this is not 100% precise
because collisions are possible. For known collisions possible feature names
are separated by | sign. Keep in mind the collision list is not exhaustive.
Features marked with (-) should be read as inverted: if they have positive
coefficient, the result is negative, if they have negative coefficient,
the result is positive.
""".lstrip()

DESCRIPTION_CLF_MULTICLASS = """
Features with largest coefficients per class.
""" + LINEAR_CAVEATS

DESCRIPTION_CLF_BINARY = """
Features with largest coefficients.
""" + LINEAR_CAVEATS

DESCRIPTION_REGRESSION = DESCRIPTION_CLF_BINARY

DESCRIPTION_REGRESSION_MULTITARGET = """
Features with largest coefficients per target.
""" + LINEAR_CAVEATS

DESCRIPTION_RANDOM_FOREST = """
Random forest feature importances; values are numbers 0 <= x <= 1;
all values sum to 1.
"""

DESCRIPTION_DECISION_TREE = """
Decision tree feature importances; values are numbers 0 <= x <= 1;
all values sum to 1.
"""

_TOP = 20

@explain_weights.register(BaseEstimator)
@singledispatch
def explain_weights_sklearn(estimator, vec=None, top=_TOP,
                            target_names=None,
                            targets=None,
                            feature_names=None, coef_scale=None,
                            feature_re=None):
    """ Return an explanation of an estimator """
    return Explanation(
        estimator=repr(estimator),
        error="estimator %r is not supported" % estimator,
    )

@explain_weights.register(OneVsRestClassifier)
def explain_weights_ovr(ovr, **kwargs):
    estimator = ovr.estimator
    func = explain_weights.dispatch(estimator.__class__)
    return func(ovr, **kwargs)

@explain_weights_sklearn.register(OneVsRestClassifier)
def explain_weights_ovr_sklearn(ovr, **kwargs):
    # dispatch OvR to eli5.sklearn
    # if explain_prediction_sklearn is called explicitly
    estimator = ovr.estimator
    func = explain_weights_sklearn.dispatch(estimator.__class__)
    return func(ovr, **kwargs)

@explain_weights_sklearn.register(LogisticRegression)
@explain_weights_sklearn.register(LogisticRegressionCV)
@explain_weights_sklearn.register(SGDClassifier)
@explain_weights_sklearn.register(PassiveAggressiveClassifier)
@explain_weights_sklearn.register(Perceptron)
@explain_weights_sklearn.register(LinearSVC)
@explain_weights_sklearn.register(RidgeClassifier)
@explain_weights_sklearn.register(RidgeClassifierCV)
def explain_linear_classifier_weights(clf,
                                      vec=None,
                                      top=_TOP,
                                      target_names=None,
                                      targets=None,
                                      feature_names=None,
                                      coef_scale=None,
                                      feature_re=None
                                      ):
    """
    Return an explanation of a linear classifier weights.

    See :func:`eli5.explain_weights` for description of
    ``top``, ``target_names``, ``targets``, ``feature_names`` and
    ``feature_re`` parameters.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the classifier ``clf``
    (e.g. a fitted CountVectorizer instance); you can pass it
    instead of ``feature_names``.

    ``coef_scale`` is a 1D np.ndarray with a scaling coefficient
    for each feature; coef[i] = coef[i] * coef_scale[i] if
    coef_scale[i] is not nan. Use it if you want to scale coefficients
    before displaying them, to take input feature sign or scale in account.
    """
    feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
                                                   coef_scale)
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)

    _extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''

    def _features(label_id):
        coef = get_coef(clf, label_id, scale=coef_scale)
        if feature_re is not None:
            coef = coef[flt_indices]
        return get_top_features(feature_names, coef, top)

    display_names = get_target_display_names(clf.classes_, target_names, targets)
    if is_multiclass_classifier(clf):
        return Explanation(
            targets=[
                TargetExplanation(
                    target=label,
                    feature_weights=_features(label_id)
                )
                for label_id, label in display_names
                ],
            description=DESCRIPTION_CLF_MULTICLASS + _extra_caveats,
            estimator=repr(clf),
            method='linear model',
        )
    else:
        # for binary classifiers scikit-learn stores a single coefficient
        # vector, which corresponds to clf.classes_[1].
        return Explanation(
            targets=[
                TargetExplanation(
                    target=display_names[1][1],
                    feature_weights=_features(0),
                )
            ],
            description=DESCRIPTION_CLF_BINARY + _extra_caveats,
            estimator=repr(clf),
            method='linear model',
        )

@explain_weights_sklearn.register(RandomForestClassifier)
@explain_weights_sklearn.register(RandomForestRegressor)
@explain_weights_sklearn.register(ExtraTreesClassifier)
@explain_weights_sklearn.register(ExtraTreesRegressor)
@explain_weights_sklearn.register(GradientBoostingClassifier)
@explain_weights_sklearn.register(GradientBoostingRegressor)
@explain_weights_sklearn.register(AdaBoostClassifier)
@explain_weights_sklearn.register(AdaBoostRegressor)
def explain_rf_feature_importance(clf,
                                  vec=None,
                                  top=_TOP,
                                  target_names=None,  # ignored
                                  targets=None,       # ignored
                                  feature_names=None,
                                  feature_re=None):
    """
    Return an explanation of a tree-based ensemble estimator.

    See :func:`eli5.explain_weights` for description of
    ``top``, ``feature_names`` and ``feature_re`` parameters.

    ``target_names`` and ``targets`` parameters are ignored.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the estimator (e.g. a fitted
    CountVectorizer instance); you can pass it instead of ``feature_names``.
    """
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    coef = clf.feature_importances_
    trees = np.array(clf.estimators_).ravel()
    coef_std = np.std([tree.feature_importances_ for tree in trees], axis=0)

    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
        coef = coef[flt_indices]
        coef_std = coef_std[flt_indices]

    indices = argsort_k_largest_positive(coef, top)
    names, values, std = feature_names[indices], coef[indices], coef_std[indices]
    return Explanation(
        feature_importances=FeatureImportances(
            [FeatureWeight(*x) for x in zip(names, values, std)],
            remaining=np.count_nonzero(coef) - len(indices),
        ),
        description=DESCRIPTION_RANDOM_FOREST,
        estimator=repr(clf),
        method='feature importances',
    )

@explain_weights_sklearn.register(DecisionTreeClassifier)
@explain_weights_sklearn.register(DecisionTreeRegressor)
def explain_decision_tree(clf,
                          vec=None,
                          top=_TOP,
                          target_names=None,
                          targets=None,  # ignored
                          feature_names=None,
                          feature_re=None,
                          **export_graphviz_kwargs):
    """
    Return an explanation of a decision tree.

    See :func:`eli5.explain_weights` for description of
    ``top``, ``target_names``, ``feature_names`` and ``feature_re`` parameters.

    ``targets`` parameter is ignored.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the estimator (e.g. a fitted
    CountVectorizer instance); you can pass it instead of ``feature_names``.

    All other keyword arguments are passed to
    `sklearn.tree.export_graphviz`_ function.

    .. _sklearn.tree.export_graphviz: http://scikit-learn.org/stable/modules/generated/sklearn.tree.export_graphviz.html
    """
    feature_names = get_feature_names(clf, vec, feature_names=feature_names)
    coef = clf.feature_importances_
    tree_feature_names = feature_names
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
        coef = coef[flt_indices]
    indices = argsort_k_largest_positive(coef, top)
    names, values = feature_names[indices], coef[indices]
    export_graphviz_kwargs.setdefault("proportion", True)
    tree_info = get_tree_info(
        clf,
        feature_names=tree_feature_names,
        class_names=target_names,
        **export_graphviz_kwargs)

    return Explanation(
        feature_importances=FeatureImportances(
            [FeatureWeight(*x) for x in zip(names, values)],
            remaining=np.count_nonzero(coef) - len(indices),
        ),
        decision_tree=tree_info,
        description=DESCRIPTION_DECISION_TREE,
        estimator=repr(clf),
        method='decision tree',
    )

@explain_weights_sklearn.register(ElasticNet)
@explain_weights_sklearn.register(ElasticNetCV)
@explain_weights_sklearn.register(HuberRegressor)
@explain_weights_sklearn.register(Lars)
@explain_weights_sklearn.register(LassoCV)
@explain_weights_sklearn.register(LinearRegression)
@explain_weights_sklearn.register(LinearSVR)
@explain_weights_sklearn.register(OrthogonalMatchingPursuit)
@explain_weights_sklearn.register(OrthogonalMatchingPursuitCV)
@explain_weights_sklearn.register(PassiveAggressiveRegressor)
@explain_weights_sklearn.register(Ridge)
@explain_weights_sklearn.register(RidgeCV)
@explain_weights_sklearn.register(SGDRegressor)
@explain_weights_sklearn.register(TheilSenRegressor)
def explain_linear_regressor_weights(reg,
                                     vec=None,
                                     top=_TOP,
                                     target_names=None,
                                     targets=None,
                                     feature_names=None,
                                     coef_scale=None,
                                     feature_re=None
                                     ):
    """
    Return an explanation of a linear regressor weights.
    See :func:`eli5.explain_weights` for description of
    ``top``, ``target_names``, ``targets``, ``feature_names`` and
    ``feature_re`` parameters.

    ``vec`` is a vectorizer instance used to transform
    raw features to the input of the regressor ``reg``; you can
    pass it instead of ``feature_names``.

    ``coef_scale`` is a 1D np.ndarray with a scaling coefficient
    for each feature; coef[i] = coef[i] * coef_scale[i] if
    coef_scale[i] is not nan. Use it if you want to scale coefficients
    before displaying them, to take input feature sign or scale in account.
    """
    feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
                                                   coef_scale)
    feature_names = get_feature_names(reg, vec, feature_names=feature_names)
    if feature_re is not None:
        feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
    _extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''

    def _features(target_id):
        coef = get_coef(reg, target_id, scale=coef_scale)
        if feature_re is not None:
            coef = coef[flt_indices]
        return get_top_features(feature_names, coef, top)

    display_names = get_target_display_names(get_default_target_names(reg),
                                             target_names, targets)
    if is_multitarget_regressor(reg):
        return Explanation(
            targets=[
                TargetExplanation(
                    target=target_name,
                    feature_weights=_features(target_id)
                )
                for target_id, target_name in display_names
                ],
            description=DESCRIPTION_REGRESSION_MULTITARGET + _extra_caveats,
            estimator=repr(reg),
            method='linear model',
            is_regression=True,
        )
    else:
        return Explanation(
            targets=[TargetExplanation(
                target=display_names[0][1],
                feature_weights=_features(0),
            )],
            description=DESCRIPTION_REGRESSION + _extra_caveats,
            estimator=repr(reg),
            method='linear model',
            is_regression=True,
        )
<|endoftext|>


<commit_before>from datetime import datetime

from django.db import models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.template.defaultfilters import slugify
from django.core.urlresolvers import reverse
from django.contrib.redirects.models import Redirect

from jsonfield.fields import JSONField

from ella.core.managers import ListingManager, RelatedManager
from ella.core.cache import get_cached_object, CachedGenericForeignKey, CachedForeignKey
from ella.core.models.main import Category, Author, Source
from ella.photos.models import Photo
from ella.core.box import Box
from ella.core.conf import core_settings

def PublishableBox(publishable, box_type, nodelist, model=None):
    "add some content type info of self.target"
    if not model:
        model = publishable.content_type.model_class()
    box_class = model.box_class

    if box_class == PublishableBox:
        box_class = Box
    return box_class(publishable, box_type, nodelist, model=model)

class Publishable(models.Model):
    """
    Base class for all object that can be published in ella
    """
    box_class = staticmethod(PublishableBox)

    content_type = models.ForeignKey(ContentType)
    target = CachedGenericForeignKey('content_type', 'id')

    category = CachedForeignKey(Category, verbose_name=_('Category'))

    # Titles
    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=255)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors'))
    source = models.ForeignKey(Source, blank=True, null=True, verbose_name=_('Source'))

    # Main Photo
    photo = CachedForeignKey(Photo, blank=True, null=True, verbose_name=_('Photo'))

    # Description
    description = models.TextField(_('Description'), blank=True)

    # Publish data
    published = models.BooleanField(_('Published'))
    publish_from = models.DateTimeField(_('Publish from'), default=core_settings.PUBLISH_FROM_WHEN_EMPTY, db_index=True)
    publish_to = models.DateTimeField(_("End of visibility"), null=True, blank=True)
    static = models.BooleanField(_('static'), default=False)

    # generic JSON field to store app cpecific data
    app_data = JSONField(default='{}', blank=True, editable=False)

    class Meta:
        app_label = 'core'
        verbose_name = _('Publishable object')
        verbose_name_plural = _('Publishable objects')

    def get_absolute_url(self, domain=False):
        " Get object's URL. "
        category = self.category

        kwargs = {
            'content_type' : slugify(self.content_type.model_class()._meta.verbose_name_plural),
            'slug' : self.slug,
        }

        if self.static:
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('static_detail', kwargs=kwargs)
            else:
                url = reverse('home_static_detail', kwargs=kwargs)
        else:
            kwargs.update({
                    'year' : self.publish_from.year,
                    'month' : self.publish_from.month,
                    'day' : self.publish_from.day,
                })
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('object_detail', kwargs=kwargs)
            else:
                url = reverse('home_object_detail', kwargs=kwargs)

        if category.site_id != settings.SITE_ID or domain:
            site = get_cached_object(Site, pk=category.site_id)
            return 'http://' + site.domain + url
        return url

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_domain_url_admin_tag(self):
        if self.get_domain_url() is not None:
            #return mark_safe(u'url' % ( self.get_domain_url() ) )
            return mark_safe(u'url' % ( self.get_domain_url() ) )
        else:
            return self.get_domain_url()
    get_domain_url_admin_tag.short_description = _('URL')
    get_domain_url_admin_tag.allow_tags = True

    def save(self, **kwargs):
        self.content_type = ContentType.objects.get_for_model(self)
        if self.pk:
            old_self = Publishable.objects.get(pk=self.pk)

            old_path = old_self.get_absolute_url()
            new_path = self.get_absolute_url()

            if old_path != new_path and new_path:
                redirect, created = Redirect.objects.get_or_create(old_path=old_path, site=self.category.site)
                redirect.new_path = new_path
                redirect.save(force_update=True)
                Redirect.objects.filter(new_path=old_path).exclude(pk=redirect.pk).update(new_path=new_path)
        return super(Publishable, self).save(**kwargs)

    def delete(self):
        url = self.get_absolute_url()
        Redirect.objects.filter(new_path=url).delete()
        return super(Publishable, self).delete()

    def __unicode__(self):
        return self.title

    def is_published(self):
        "Return True if the Publishable is currently active."
        now = datetime.now()
        return self.published and now > self.publish_from and (self.publish_to is None or now < self.publish_to)

def ListingBox(listing, *args, **kwargs):
    " Delegate the boxing to the target's Box class. "
    obj = listing.publishable
    return obj.box_class(obj, *args, **kwargs)

class Listing(models.Model):
    """
    Listing of an object in a category. Each and every object that have it's own detail page must have a Listing object
    that is valid (not expired) and places him in the object's main category. Any object can be listed in any number of
    categories (but only once per category). Even if the object is listed in other categories besides its main category,
    its detail page's url still belongs to the main one.

    see doc/listing.txt for more details on Listings
    """
    box_class = staticmethod(ListingBox)

    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)

    publish_from = models.DateTimeField(_("Start of listing"), db_index=True)
    publish_to = models.DateTimeField(_("End of listing"), null=True, blank=True)

    commercial = models.BooleanField(_("Commercial"), default=False, help_text=_("Check this if the listing is of a commercial content."))

    objects = ListingManager()

    @property
    def target(self):
        return self.publishable

    def get_absolute_url(self, domain=False):
        return self.publishable.get_absolute_url(domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_publish_from(self):
        return self.publish_from

    def __unicode__(self):
        try:
            return u'%s listed in %s' % (self.publishable, self.category)
        except:
            return 'Broken listing'

    class Meta:
        app_label = 'core'
        verbose_name = _('Listing')
        verbose_name_plural = _('Listings')

class Related(models.Model):
    """
    Related objects - model for recording related items. For example related articles.
    """
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))

    related_ct = models.ForeignKey(ContentType, verbose_name=_('Content type'))
    related_id = models.IntegerField(_('Object ID'))
    related = CachedGenericForeignKey('related_ct', 'related_id')

    objects = RelatedManager()

    def __unicode__(self):
        return u'%s relates to %s' % (self.publishable, self.related)

    class Meta:
        app_label = 'core'
        verbose_name = _('Related')
        verbose_name_plural = _('Related')

<commit_msg>More docs for core models, better code readability<commit_after>from datetime import datetime

from django.db import models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.template.defaultfilters import slugify
from django.core.urlresolvers import reverse
from django.contrib.redirects.models import Redirect

from jsonfield.fields import JSONField

from ella.core.box import Box
from ella.core.conf import core_settings
from ella.core.cache import get_cached_object, CachedGenericForeignKey, \
    CachedForeignKey
from ella.core.managers import ListingManager, RelatedManager
from ella.core.models.main import Category, Author, Source
from ella.photos.models import Photo

def PublishableBox(publishable, box_type, nodelist, model=None):
    "add some content type info of self.target"
    if not model:
        model = publishable.content_type.model_class()
    box_class = model.box_class

    if box_class == PublishableBox:
        box_class = Box
    return box_class(publishable, box_type, nodelist, model=model)

class Publishable(models.Model):
    """
    Base class for all object that can be published in ella
    """
    box_class = staticmethod(PublishableBox)

    content_type = models.ForeignKey(ContentType)
    target = CachedGenericForeignKey('content_type', 'id')

    category = CachedForeignKey(Category, verbose_name=_('Category'))

    # Titles
    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=255)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors'))
    source = models.ForeignKey(Source, blank=True, null=True,
        verbose_name=_('Source'))

    # Main Photo
    photo = CachedForeignKey(Photo, blank=True, null=True, verbose_name=_('Photo'))

    # Description
    description = models.TextField(_('Description'), blank=True)

    # Publish data
    published = models.BooleanField(_('Published'))
    publish_from = models.DateTimeField(_('Publish from'),
        default=core_settings.PUBLISH_FROM_WHEN_EMPTY, db_index=True)
    publish_to = models.DateTimeField(_("End of visibility"), null=True, blank=True)
    static = models.BooleanField(_('static'), default=False)

    # generic JSON field to store app cpecific data
    app_data = JSONField(default='{}', blank=True, editable=False)

    class Meta:
        app_label = 'core'
        verbose_name = _('Publishable object')
        verbose_name_plural = _('Publishable objects')

    def __unicode__(self):
        return self.title

    def get_absolute_url(self, domain=False):
        " Get object's URL. "
        category = self.category

        kwargs = {
            'content_type' : slugify(self.content_type.model_class()._meta.verbose_name_plural),
            'slug' : self.slug,
        }

        if self.static:
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('static_detail', kwargs=kwargs)
            else:
                url = reverse('home_static_detail', kwargs=kwargs)
        else:
            kwargs.update({
                    'year' : self.publish_from.year,
                    'month' : self.publish_from.month,
                    'day' : self.publish_from.day,
                })
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('object_detail', kwargs=kwargs)
            else:
                url = reverse('home_object_detail', kwargs=kwargs)

        if category.site_id != settings.SITE_ID or domain:
            site = get_cached_object(Site, pk=category.site_id)
            return 'http://' + site.domain + url
        return url

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_domain_url_admin_tag(self):
        if self.get_domain_url() is not None:
            return mark_safe(u'url' % (self.get_domain_url()))
        else:
            return self.get_domain_url()
    get_domain_url_admin_tag.short_description = _('URL')
    get_domain_url_admin_tag.allow_tags = True

    def save(self, **kwargs):
        self.content_type = ContentType.objects.get_for_model(self)
        if self.pk:
            old_self = Publishable.objects.get(pk=self.pk)

            old_path = old_self.get_absolute_url()
            new_path = self.get_absolute_url()

            if old_path != new_path and new_path:
                redirect = Redirect.objects.get_or_create(old_path=old_path,
                    site=self.category.site)[0]
                redirect.new_path = new_path
                redirect.save(force_update=True)
                Redirect.objects.filter(new_path=old_path).exclude(
                    pk=redirect.pk).update(new_path=new_path)
        return super(Publishable, self).save(**kwargs)

    def delete(self):
        url = self.get_absolute_url()
        Redirect.objects.filter(new_path=url).delete()
        return super(Publishable, self).delete()

    def is_published(self):
        "Return True if the Publishable is currently active."
        now = datetime.now()
        return self.published and now > self.publish_from and \
            (self.publish_to is None or now < self.publish_to)

def ListingBox(listing, *args, **kwargs):
    " Delegate the boxing to the target's Box class. "
    obj = listing.publishable
    return obj.box_class(obj, *args, **kwargs)

class Listing(models.Model):
    """
    Listing of an ``Publishable`` in a ``Category``. Each and every object that have it's
    own detail page must have a ``Listing`` object that is valid (not expired) and
    places him in the object's main category. Any object can be listed in any
    number of categories (but only once per category). Even if the object is
    listed in other categories besides its main category, its detail page's url
    still belongs to the main one.
    """
    box_class = staticmethod(ListingBox)

    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)

    publish_from = models.DateTimeField(_("Start of listing"), db_index=True)
    publish_to = models.DateTimeField(_("End of listing"), null=True, blank=True)

    commercial = models.BooleanField(_("Commercial"), default=False,
        help_text=_("Check this if the listing is of a commercial content."))

    objects = ListingManager()

    class Meta:
        app_label = 'core'
        verbose_name = _('Listing')
        verbose_name_plural = _('Listings')

    def __unicode__(self):
        try:
            return _(u'%s listed in %s') % (self.publishable, self.category)
        except:
            return _('Broken listing')

    @property
    def target(self):
        return self.publishable

    def get_absolute_url(self, domain=False):
        return self.publishable.get_absolute_url(domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_publish_from(self):
        return self.publish_from

class Related(models.Model):
    """
    Related objects - model for recording related ``Publishable`` objects.

    An example would be two articles sharing a similar topic. When something
    like this happens, a ``Related`` instance connecting the objects should
    be created.
    """
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))

    related_ct = models.ForeignKey(ContentType, verbose_name=_('Content type'))
    related_id = models.IntegerField(_('Object ID'))
    related = CachedGenericForeignKey('related_ct', 'related_id')

    objects = RelatedManager()

    class Meta:
        app_label = 'core'
        verbose_name = _('Related')
        verbose_name_plural = _('Related')

    def __unicode__(self):
        return _(u'%s relates to %s') % (self.publishable, self.related)

<|endoftext|>