<commit_before>#!/usr/bin/env python
from collections import defaultdict
import GeminiQuery
import sql_utils
from mendelianerror import mendelian_error
from gemini_constants import *
import gemini_subjects as subjects

def get_prob(family_gt_lls, row):
    e = {}
    for k in ('gt_phred_ll_homref', 'gt_phred_ll_het', 'gt_phred_ll_homalt'):
        e[k] = row[k]
    for k, li in family_gt_lls.iteritems():
        for i in range(len(li)):
            if isinstance(li[i], basestring) or li[i].__class__ == "code":
                li[i] = eval(li[i], e)
        # order is father, mother, child
    father = [family_gt_lls[k][0] for k in ("homref", "het", "homalt")]
    mother = [family_gt_lls[k][1] for k in ("homref", "het", "homalt")]
    child = [family_gt_lls[k][2] for k in ("homref", "het", "homalt")]
    return mendelian_error(father, mother, child, pls=True)

class GeminiInheritanceModelFactory(object):

    gt_cols = ('gts', 'gt_types', 'gt_phases', 'gt_depths', 'gt_ref_depths',
               'gt_alt_depths', 'gt_quals', 'gt_phred_ll_homref',
               'gt_phred_ll_het', 'gt_phred_ll_homalt')

    # https://github.com/arq5x/gemini/pull/436
    required_columns = ("variant_id", "family_id", "family_members",
                        "family_genotypes", "samples", "family_count")

    def __init__(self, args, model):

        # default to all genotype columns and all columns
        if not args.columns:
            args.columns = "*," + ", ".join(self.gt_cols)

        self.args = args
        self.model = model
        self.gq = GeminiQuery.GeminiQuery(args.db, include_gt_cols=True)

    def get_candidates(self):
        """
        Report candidate variants that meet the requested inheritance model.
        """
        if self.model in ["auto_dom", "auto_rec"] \
           or (self.model == "de_novo" and self.args.min_kindreds is not None):
            self._get_gene_only_candidates()
        else:
            self._get_all_candidates()

    @classmethod
    def report_candidates(self, candidates, is_violation=False,
                          min_kindreds=1, is_comp_het=False):
        """
        Print variants that meet the user's requirements
        If input is a tuple,
        """
        family_count = len(candidates)

        if family_count < min_kindreds:
            return False

        candidate_keys = sorted(candidates.keys())
        fam_counts_by_variant = defaultdict(int)
        for (variant_id, gene), li in candidates.items():
            fam_counts_by_variant[variant_id] += len(li)

        for (variant_id, gene) in candidate_keys:
            for tup in candidates[(variant_id, gene)]:

                # (row, family_gt_label, family_gt_cols) \
                violation = ""
                comp_het = ""
                if is_violation:
                    (row, family_gt_label, family_gt_cols, family_id,
                            family_gt_lls, violation) = tuple(tup)
                elif is_comp_het:
                    (row, family_gt_label, family_gt_cols, family_id, comp_het) = tuple(tup)
                else:
                    (row, family_gt_label, family_gt_cols, family_id) = tuple(tup)

                e = {}
                for k in ('gt_types', 'gts', 'gt_depths'):
                    e[k] = row[k]

                if is_violation:
                    prob = get_prob(family_gt_lls, row)
                    violation += ("\t%.3f" % prob)

                v_id = False
                if 'variant_id' in row.row:
                    row.row.pop('variant_id')
                    v_id = True
                affected_samples = [x.split("(")[0] for x in family_gt_label if ";affected" in x]

                print str(row) + "\t%s\t%s\t%s\t%s\t%s\t%i%s" % (variant_id,
                             family_id,
                             ",".join(str(s) for s in family_gt_label),
                             ",".join(str(eval(s, e)) for s in family_gt_cols),
                             ",".join(affected_samples),
                             fam_counts_by_variant[variant_id],
                             ("\t" + violation + comp_het).rstrip())
                if v_id:
                    row.row['variant_id'] = variant_id

    def _cull_families(self):
        """
        If the user has asked to restric the analysis to a specific set
        of families, then we need to prune the list of possible families
        to that specific subset.
        """

    def _get_family_info(self):
        """
        Extract the relevant genotype filters, as well all labels
        for each family in the database.
        """
        families = subjects.get_families(self.args.db, self.args.families)
        self.family_ids = []
        self.family_masks = []
        self.family_gt_labels = []
        self.family_gt_columns = []
        self.family_dp_columns = []
        self.family_gt_phred_lls = []
        for family in families:

            family_filter = None

            if self.model == "auto_rec":
                family_filter = family.get_auto_recessive_filter(gt_ll=self.args.gt_phred_ll)
            elif self.model == "auto_dom":
                family_filter = family.get_auto_dominant_filter(gt_ll=self.args.gt_phred_ll)
            elif self.model == "de_novo":
                family_filter = family.get_de_novo_filter(self.args.only_affected, gt_ll=self.args.gt_phred_ll)
            elif self.model == "mendel_violations":
                family_filter = family.get_mendelian_violation_filter(gt_ll=self.args.gt_phred_ll)

            self.family_masks.append(family_filter)
            self.family_gt_labels.append(family.get_genotype_labels())
            self.family_gt_columns.append(family.get_genotype_columns())
            self.family_dp_columns.append(family.get_genotype_depths())
            self.family_ids.append(family.family_id)
            if self.model == "mendel_violations":
                self.family_gt_phred_lls.append(family.get_genotype_lls())

    def _construct_query(self):
        """
        Construct the relevant query based on the user's requests.
        """
        if self.args.columns is not None:
            # the user only wants to report a subset of the columns
            self.query = "SELECT " + str(self.args.columns) + " FROM variants "
        else:
            # report the kitchen sink
            self.query = "SELECT chrom, start, end, * \
                    , gts, gt_types, gt_phases, gt_depths, \
                    gt_ref_depths, gt_alt_depths, gt_quals, \
                    gt_phred_ll_homref, gt_phred_ll_het, gt_phred_ll_homalt \
                    FROM variants "

        # add any non-genotype column limits to the where clause
        if self.args.filter:
            self.query += " WHERE " + self.args.filter

        # auto_rec and auto_dom candidates should be limited to
        # variants affecting genes.
        if self.model == "auto_rec" or self.model == "auto_dom"\
        or (self.model == "de_novo" and self.args.min_kindreds is not None):

            # we require the "gene" column for the auto_* tools
            self.query = sql_utils.ensure_columns(self.query, ['gene'])
            if self.args.filter:
                self.query += " AND gene is not NULL ORDER BY chrom, gene"
            else:
                self.query += " WHERE gene is not NULL ORDER BY chrom, gene"
        self.query = sql_utils.ensure_columns(self.query, ['variant_id'])

    @classmethod
    def get_header(cls, gqh, is_violation_query, is_comp_het=False):
        h = "\t".join(cls.required_columns)

        # strip variant_id as they didn't request it, but we added it for the
        # required columns
        if gqh.endswith("\tvariant_id"):
            gqh, _ = gqh.rsplit("\t", 1)

        header = gqh + "\t" + h
        if is_violation_query:
            return header + "\tviolation\tviolation_prob"
        elif is_comp_het:
            return header + "\tcomp_het_id"
        else:
            return header

    def _get_gene_only_candidates(self):
        """
        Identify candidates that meet the user's criteria AND affect genes.
        """
        # collect family info
        self._get_family_info()

        # run the query applying any genotype filters provided by the user.
        self._construct_query()
        self.gq.run(self.query, needs_genotypes=True)

        is_violation_query = isinstance(self.family_masks[0], dict)
        print self.get_header(self.gq.header, is_violation_query)

        # yield the resulting variants for this familiy
        self.candidates = defaultdict(list)
        prev_gene = None
        for row in self.gq:

            curr_gene = row['gene']
            variant_id = row.row.pop('variant_id')

            # report any candidates for the previous gene
            if curr_gene != prev_gene and prev_gene is not None:
                self.report_candidates(self.candidates, is_violation_query,
                        self.args.min_kindreds)
                # reset for the next gene
                self.candidates = defaultdict(list)

            # test the variant for each family in the db
            for idx, fam_id in enumerate(self.family_ids):
                family_genotype_mask = self.family_masks[idx]
                family_gt_labels = self.family_gt_labels[idx]
                family_gt_cols = self.family_gt_columns[idx]
                family_dp_cols = self.family_dp_columns[idx]

                # interrogate the genotypes present in each family member to
                # conforming to the genetic model being tested
                for c in ('gt_types', 'gts', 'gt_depths', 'gt_phred_ll_homalt',
                          'gt_phred_ll_het', 'gt_phred_ll_homref'):
                    e[c] = row[c]

                if is_violation_query:
                    family_gt_phred_lls = self.family_gt_phred_lls[idx]

                # skip if the variant doesn't meet a recessive model
                # for this family
                violations = []
                if is_violation_query:
                    for violation, mask in family_genotype_mask.items():
                        if eval(mask, e):
                            violations.append(violation)
                    if len(violations) == 0:
                        continue
                else:
                    if not eval(family_genotype_mask, e):
                        continue

                # make sure each sample's genotype had sufficient coverage.
                # otherwise, ignore
                insufficient_depth = False
                for col in family_dp_cols:
                    depth = int(eval(col, e))
                    if depth < self.args.min_sample_depth:
                        insufficient_depth = True
                        break
                if insufficient_depth is True:
                    continue

                # if it meets a recessive model, add it to the list
                # of candidates for this gene.
                self.candidates[(variant_id, curr_gene)].append([row,
                                                        family_gt_labels,
                                                        family_gt_cols,
                                                        fam_id])
                if is_violation_query:
                    self.candidates[(variant_id, curr_gene)][-1].append(family_gt_phred_lls)
                    self.candidates[(variant_id, curr_gene)][-1].append(",".join(violations))

            prev_gene = curr_gene

        # report any candidates for the last gene
        self.report_candidates(self.candidates, is_violation_query,
                self.args.min_kindreds)

    def _get_all_candidates(self):
        """
        Identify candidates that meet the user's criteria no matter where
        they occur in the genome.
        """
        """
        Identify candidates that meet the user's criteria AND affect genes.
        """
        # collect family info
        self._get_family_info()

        # run the query applying any genotype filters provided by the user.
        self._construct_query()
        self.gq.run(self.query, needs_genotypes=True)

        is_violation_query = isinstance(self.family_masks[0], dict)
        print self.get_header(self.gq.header, is_violation_query)
        gene = None

        for row in self.gq:
            variant_id = row.row.pop('variant_id')
            candidates = defaultdict(list)
            cols = {}
            for col in self.gt_cols:
                cols[col] = row[col]

            # test the variant for each family in the db
            for idx, family_id in enumerate(self.family_ids):
                family_genotype_mask = self.family_masks[idx]
                family_gt_labels = self.family_gt_labels[idx]
                family_gt_cols = self.family_gt_columns[idx]
                family_dp_cols = self.family_dp_columns[idx]
                if is_violation_query:
                    family_gt_phred_lls = self.family_gt_phred_lls[idx]

                # interrogate the genotypes present in each family member to
                # conforming to the genetic model being tested

                # skip if the variant doesn't meet a recessive model
                # for this family
                violations = []
                if is_violation_query:
                    for violation, mask in family_genotype_mask.items():
                        if eval(mask, cols):
                            violations.append(violation)
                    if len(violations) == 0:
                        continue
                else:
                    if not eval(family_genotype_mask, cols):
                        continue

                # make sure each sample's genotype had sufficient coverage.
                # otherwise, ignore
                insufficient_depth = False
                for col in family_dp_cols:
                    depth = int(eval(col, cols))
                    if depth < self.args.min_sample_depth:
                        insufficient_depth = True
                        break
                if insufficient_depth is True:
                    continue

                # shoe-horn the variant so we can use report_candidates.
                key = (variant_id, gene)
                candidates[key].append([row, family_gt_labels, family_gt_cols, family_id])
                if is_violation_query:
                    candidates[key][-1].append(family_gt_phred_lls)
                    candidates[key][-1].append(",".join(violations))
            self.report_candidates(candidates, is_violation_query,
                                   self.args.min_kindreds)

<commit_msg>fix printing<commit_after>#!/usr/bin/env python
from collections import defaultdict
import GeminiQuery
import sql_utils
from mendelianerror import mendelian_error
from gemini_constants import *
import gemini_subjects as subjects

def get_prob(family_gt_lls, row):
    e = {}
    for k in ('gt_phred_ll_homref', 'gt_phred_ll_het', 'gt_phred_ll_homalt'):
        e[k] = row[k]
    for k, li in family_gt_lls.iteritems():
        for i in range(len(li)):
            if isinstance(li[i], basestring) or li[i].__class__ == "code":
                li[i] = eval(li[i], e)
        # order is father, mother, child
    father = [family_gt_lls[k][0] for k in ("homref", "het", "homalt")]
    mother = [family_gt_lls[k][1] for k in ("homref", "het", "homalt")]
    child = [family_gt_lls[k][2] for k in ("homref", "het", "homalt")]
    return mendelian_error(father, mother, child, pls=True)

class GeminiInheritanceModelFactory(object):

    gt_cols = ('gts', 'gt_types', 'gt_phases', 'gt_depths', 'gt_ref_depths',
               'gt_alt_depths', 'gt_quals', 'gt_phred_ll_homref',
               'gt_phred_ll_het', 'gt_phred_ll_homalt')

    # https://github.com/arq5x/gemini/pull/436
    required_columns = ("family_id", "family_members",
                        "family_genotypes", "samples", "family_count")

    def __init__(self, args, model):

        # default to all genotype columns and all columns
        if not args.columns:
            args.columns = "*," + ", ".join(self.gt_cols)

        self.args = args
        self.model = model
        self.gq = GeminiQuery.GeminiQuery(args.db, include_gt_cols=True)

    def get_candidates(self):
        """
        Report candidate variants that meet the requested inheritance model.
        """
        if self.model in ["auto_dom", "auto_rec"] \
           or (self.model == "de_novo" and self.args.min_kindreds is not None):
            self._get_gene_only_candidates()
        else:
            self._get_all_candidates()

    @classmethod
    def report_candidates(self, candidates, is_violation=False,
                          min_kindreds=1, is_comp_het=False):
        """
        Print variants that meet the user's requirements
        If input is a tuple,
        """
        family_count = len(candidates)

        if family_count < min_kindreds:
            return False

        candidate_keys = sorted(candidates.keys())
        fam_counts_by_variant = defaultdict(int)
        for (variant_id, gene), li in candidates.items():
            fam_counts_by_variant[variant_id] += len(li)

        for (variant_id, gene) in candidate_keys:
            for tup in candidates[(variant_id, gene)]:

                # (row, family_gt_label, family_gt_cols) \
                violation = ""
                comp_het = ""
                if is_violation:
                    (row, family_gt_label, family_gt_cols, family_id,
                            family_gt_lls, violation) = tuple(tup)
                elif is_comp_het:
                    (row, family_gt_label, family_gt_cols, family_id, comp_het) = tuple(tup)
                else:
                    (row, family_gt_label, family_gt_cols, family_id) = tuple(tup)

                e = {}
                for k in ('gt_types', 'gts', 'gt_depths'):
                    e[k] = row[k]

                if is_violation:
                    prob = get_prob(family_gt_lls, row)
                    violation += ("\t%.3f" % prob)

                affected_samples = [x.split("(")[0] for x in family_gt_label if ";affected" in x]

                print str(row) + "\t%s\t%s\t%s\t%s\t%s\t%i%s" % (variant_id,
                             family_id,
                             ",".join(str(s) for s in family_gt_label),
                             ",".join(str(eval(s, e)) for s in family_gt_cols),
                             ",".join(affected_samples),
                             fam_counts_by_variant[variant_id],
                             ("\t" + violation + comp_het).rstrip())

    def _cull_families(self):
        """
        If the user has asked to restric the analysis to a specific set
        of families, then we need to prune the list of possible families
        to that specific subset.
        """

    def _get_family_info(self):
        """
        Extract the relevant genotype filters, as well all labels
        for each family in the database.
        """
        families = subjects.get_families(self.args.db, self.args.families)
        self.family_ids = []
        self.family_masks = []
        self.family_gt_labels = []
        self.family_gt_columns = []
        self.family_dp_columns = []
        self.family_gt_phred_lls = []
        for family in families:

            family_filter = None

            if self.model == "auto_rec":
                family_filter = family.get_auto_recessive_filter(gt_ll=self.args.gt_phred_ll)
            elif self.model == "auto_dom":
                family_filter = family.get_auto_dominant_filter(gt_ll=self.args.gt_phred_ll)
            elif self.model == "de_novo":
                family_filter = family.get_de_novo_filter(self.args.only_affected, gt_ll=self.args.gt_phred_ll)
            elif self.model == "mendel_violations":
                family_filter = family.get_mendelian_violation_filter(gt_ll=self.args.gt_phred_ll)

            self.family_masks.append(family_filter)
            self.family_gt_labels.append(family.get_genotype_labels())
            self.family_gt_columns.append(family.get_genotype_columns())
            self.family_dp_columns.append(family.get_genotype_depths())
            self.family_ids.append(family.family_id)
            if self.model == "mendel_violations":
                self.family_gt_phred_lls.append(family.get_genotype_lls())

    def _construct_query(self):
        """
        Construct the relevant query based on the user's requests.
        """
        if self.args.columns is not None:
            # the user only wants to report a subset of the columns
            self.query = "SELECT " + str(self.args.columns) + " FROM variants "
        else:
            # report the kitchen sink
            self.query = "SELECT chrom, start, end, * \
                    , gts, gt_types, gt_phases, gt_depths, \
                    gt_ref_depths, gt_alt_depths, gt_quals, \
                    gt_phred_ll_homref, gt_phred_ll_het, gt_phred_ll_homalt \
                    FROM variants "

        # add any non-genotype column limits to the where clause
        if self.args.filter:
            self.query += " WHERE " + self.args.filter

        # auto_rec and auto_dom candidates should be limited to
        # variants affecting genes.
        if self.model == "auto_rec" or self.model == "auto_dom"\
        or (self.model == "de_novo" and self.args.min_kindreds is not None):

            # we require the "gene" column for the auto_* tools
            self.query = sql_utils.ensure_columns(self.query, ['gene'])
            if self.args.filter:
                self.query += " AND gene is not NULL ORDER BY chrom, gene"
            else:
                self.query += " WHERE gene is not NULL ORDER BY chrom, gene"
        self.query = sql_utils.ensure_columns(self.query, ['variant_id'])

    @classmethod
    def get_header(cls, gqh, is_violation_query, is_comp_het=False):
        h = "\t".join(cls.required_columns)

        header = gqh + "\t" + h
        if is_violation_query:
            return header + "\tviolation\tviolation_prob"
        elif is_comp_het:
            return header + "\tcomp_het_id"
        else:
            return header

    def _get_gene_only_candidates(self):
        """
        Identify candidates that meet the user's criteria AND affect genes.
        """
        # collect family info
        self._get_family_info()

        # run the query applying any genotype filters provided by the user.
        self._construct_query()
        self.gq.run(self.query, needs_genotypes=True)

        is_violation_query = isinstance(self.family_masks[0], dict)
        print self.get_header(self.gq.header, is_violation_query)

        # yield the resulting variants for this familiy
        self.candidates = defaultdict(list)
        prev_gene = None
        for row in self.gq:

            curr_gene = row['gene']
            variant_id = row.row['variant_id']

            # report any candidates for the previous gene
            if curr_gene != prev_gene and prev_gene is not None:
                self.report_candidates(self.candidates, is_violation_query,
                        self.args.min_kindreds)
                # reset for the next gene
                self.candidates = defaultdict(list)

            # test the variant for each family in the db
            for idx, fam_id in enumerate(self.family_ids):
                family_genotype_mask = self.family_masks[idx]
                family_gt_labels = self.family_gt_labels[idx]
                family_gt_cols = self.family_gt_columns[idx]
                family_dp_cols = self.family_dp_columns[idx]

                # interrogate the genotypes present in each family member to
                # conforming to the genetic model being tested
                for c in ('gt_types', 'gts', 'gt_depths', 'gt_phred_ll_homalt',
                          'gt_phred_ll_het', 'gt_phred_ll_homref'):
                    e[c] = row[c]

                if is_violation_query:
                    family_gt_phred_lls = self.family_gt_phred_lls[idx]

                # skip if the variant doesn't meet a recessive model
                # for this family
                violations = []
                if is_violation_query:
                    for violation, mask in family_genotype_mask.items():
                        if eval(mask, e):
                            violations.append(violation)
                    if len(violations) == 0:
                        continue
                else:
                    if not eval(family_genotype_mask, e):
                        continue

                # make sure each sample's genotype had sufficient coverage.
                # otherwise, ignore
                insufficient_depth = False
                for col in family_dp_cols:
                    depth = int(eval(col, e))
                    if depth < self.args.min_sample_depth:
                        insufficient_depth = True
                        break
                if insufficient_depth is True:
                    continue

                # if it meets a recessive model, add it to the list
                # of candidates for this gene.
                self.candidates[(variant_id, curr_gene)].append([row,
                                                        family_gt_labels,
                                                        family_gt_cols,
                                                        fam_id])
                if is_violation_query:
                    self.candidates[(variant_id, curr_gene)][-1].append(family_gt_phred_lls)
                    self.candidates[(variant_id, curr_gene)][-1].append(",".join(violations))

            prev_gene = curr_gene

        # report any candidates for the last gene
        self.report_candidates(self.candidates, is_violation_query,
                self.args.min_kindreds)

    def _get_all_candidates(self):
        """
        Identify candidates that meet the user's criteria no matter where
        they occur in the genome.
        """
        """
        Identify candidates that meet the user's criteria AND affect genes.
        """
        # collect family info
        self._get_family_info()

        # run the query applying any genotype filters provided by the user.
        self._construct_query()
        self.gq.run(self.query, needs_genotypes=True)

        is_violation_query = isinstance(self.family_masks[0], dict)
        print self.get_header(self.gq.header, is_violation_query)
        gene = None

        for row in self.gq:
            variant_id = row.row['variant_id']
            candidates = defaultdict(list)
            cols = {}
            for col in self.gt_cols:
                cols[col] = row[col]

            # test the variant for each family in the db
            for idx, family_id in enumerate(self.family_ids):
                family_genotype_mask = self.family_masks[idx]
                family_gt_labels = self.family_gt_labels[idx]
                family_gt_cols = self.family_gt_columns[idx]
                family_dp_cols = self.family_dp_columns[idx]
                if is_violation_query:
                    family_gt_phred_lls = self.family_gt_phred_lls[idx]

                # interrogate the genotypes present in each family member to
                # conforming to the genetic model being tested

                # skip if the variant doesn't meet a recessive model
                # for this family
                violations = []
                if is_violation_query:
                    for violation, mask in family_genotype_mask.items():
                        if eval(mask, cols):
                            violations.append(violation)
                    if len(violations) == 0:
                        continue
                else:
                    if not eval(family_genotype_mask, cols):
                        continue

                # make sure each sample's genotype had sufficient coverage.
                # otherwise, ignore
                insufficient_depth = False
                for col in family_dp_cols:
                    depth = int(eval(col, cols))
                    if depth < self.args.min_sample_depth:
                        insufficient_depth = True
                        break
                if insufficient_depth is True:
                    continue

                # shoe-horn the variant so we can use report_candidates.
                key = (variant_id, gene)
                candidates[key].append([row, family_gt_labels, family_gt_cols, family_id])
                if is_violation_query:
                    candidates[key][-1].append(family_gt_phred_lls)
                    candidates[key][-1].append(",".join(violations))
            self.report_candidates(candidates, is_violation_query,
                                   self.args.min_kindreds)

<|endoftext|>


<commit_before>from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed
from django.utils import timezone
from mezzanine.conf import settings

from .models import Article
from ..album.models import Album
from ..faces.models import Face
from ..resources.models import Resource, Factoid
from ..news.models import NewsPost

import itertools
import datetime

days_ago = int(settings.FEED_GENERATION_DAYS)

class BaseFeed(Feed):
    def __call__(self, request, *args, **kwargs):
        accept_header = request.META.get("HTTP_ACCEPT", "")
        if "application/atom+xml".find(accept_header) >= 0:
            self.feed_type = Atom1Feed
        else:
            self.feed_type = Rss201rev2Feed
        return super(BaseFeed, self).__call__(request, *args, **kwargs)

    def item_pubdate(self, item):
        return item.publish_date

    def item_author_name(self, item):
        author = getattr(item, 'author', None)
        if author:
            return author.title
        user = getattr(item, 'user', None)
        if user:
            return item.user.get_full_name() or item.user.username

    def item_author_link(self, item):
        author = getattr(item, 'author', None)
        if author:
            return author.get_absolute_url()

class AllFeed(BaseFeed):
    title = "PARI consolidated feed"
    link  = "/feeds/all/"
    description = "Updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return itertools.chain(
            Article.objects.filter(publish_date__gte=x_days_ago),
            Album.objects.filter(publish_date__gte=x_days_ago),
            Face.objects.filter(publish_date__gte=x_days_ago),
            Resource.objects.filter(publish_date__gte=x_days_ago),
            Factoid.objects.filter(publish_date__gte=x_days_ago),
            NewsPost.objects.filter(publish_date__gte=x_days_ago)
        )

class ArticleFeed(BaseFeed):
    title = "PARI article feed"
    link  = "/feeds/articles/"
    description = "Article updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Article.objects.filter(publish_date__gte=x_days_ago)

class AlbumFeed(BaseFeed):
    title = "PARI album feed"
    link  = "/feeds/albums/"
    description = "Album updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Album.objects.filter(publish_date__gte=x_days_ago)

class FaceFeed(BaseFeed):
    title = "PARI face feed"
    link  = "/feeds/faces/"
    description = "Face updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Face.objects.filter(publish_date__gte=x_days_ago)

class ResourceFeed(BaseFeed):
    title = "PARI resource feed"
    link  = "/feeds/resources/"
    description = "Resource updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Resource.objects.filter(publish_date__gte=x_days_ago)

class FactoidFeed(BaseFeed):
    title = "PARI factoid feed"
    link  = "/feeds/factoids/"
    description = "Factoid updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Factoid.objects.filter(publish_date__gte=x_days_ago)

class NewsPostFeed(BaseFeed):
    title = "PARI news feed"
    link  = "/feeds/newsposts/"
    description = "News updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return NewsPost.objects.filter(publish_date__gte=x_days_ago)
<commit_msg>Fix a feed test<commit_after>from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed
from django.utils import timezone
from mezzanine.conf import settings

from .models import Article
from ..album.models import Album
from ..faces.models import Face
from ..resources.models import Resource, Factoid
from ..news.models import NewsPost

import itertools
import datetime

days_ago = int(settings.FEED_GENERATION_DAYS)

class BaseFeed(Feed):
    def __call__(self, request, *args, **kwargs):
        accept_header = request.META.get("HTTP_ACCEPT", "")
        if accept_header.find("application/atom+xml") >= 0:
            self.feed_type = Atom1Feed
        else:
            self.feed_type = Rss201rev2Feed
        return super(BaseFeed, self).__call__(request, *args, **kwargs)

    def item_pubdate(self, item):
        return item.publish_date

    def item_author_name(self, item):
        author = getattr(item, 'author', None)
        if author:
            return author.title
        user = getattr(item, 'user', None)
        if user:
            return item.user.get_full_name() or item.user.username

    def item_author_link(self, item):
        author = getattr(item, 'author', None)
        if author:
            return author.get_absolute_url()

class AllFeed(BaseFeed):
    title = "PARI consolidated feed"
    link  = "/feeds/all/"
    description = "Updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return itertools.chain(
            Article.objects.filter(publish_date__gte=x_days_ago),
            Album.objects.filter(publish_date__gte=x_days_ago),
            Face.objects.filter(publish_date__gte=x_days_ago),
            Resource.objects.filter(publish_date__gte=x_days_ago),
            Factoid.objects.filter(publish_date__gte=x_days_ago),
            NewsPost.objects.filter(publish_date__gte=x_days_ago)
        )

class ArticleFeed(BaseFeed):
    title = "PARI article feed"
    link  = "/feeds/articles/"
    description = "Article updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Article.objects.filter(publish_date__gte=x_days_ago)

class AlbumFeed(BaseFeed):
    title = "PARI album feed"
    link  = "/feeds/albums/"
    description = "Album updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Album.objects.filter(publish_date__gte=x_days_ago)

class FaceFeed(BaseFeed):
    title = "PARI face feed"
    link  = "/feeds/faces/"
    description = "Face updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Face.objects.filter(publish_date__gte=x_days_ago)

class ResourceFeed(BaseFeed):
    title = "PARI resource feed"
    link  = "/feeds/resources/"
    description = "Resource updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Resource.objects.filter(publish_date__gte=x_days_ago)

class FactoidFeed(BaseFeed):
    title = "PARI factoid feed"
    link  = "/feeds/factoids/"
    description = "Factoid updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return Factoid.objects.filter(publish_date__gte=x_days_ago)

class NewsPostFeed(BaseFeed):
    title = "PARI news feed"
    link  = "/feeds/newsposts/"
    description = "News updates on the PARI site over the past {0} days".format(days_ago)

    def items(self):
        x_days_ago = timezone.now() - datetime.timedelta(days=days_ago)
        return NewsPost.objects.filter(publish_date__gte=x_days_ago)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Custom serializer fields for nested translations."""

from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured

from rest_framework import serializers

from parler_rest.utils import create_translated_fields_serializer

class TranslatedFieldsField(serializers.Field):

    """Exposing translated fields for a TranslatableModel in REST style."""

    default_error_messages = dict(serializers.Field.default_error_messages, **{
        'invalid': "Input is not a valid dict",
    })

    def __init__(self, *args, **kwargs):
        """Receive custom serializer class and model."""
        self.serializer_class = kwargs.pop('serializer_class', None)
        self.shared_model = kwargs.pop('shared_model', None)
        super(TranslatedFieldsField, self).__init__(*args, **kwargs)

    def bind(self, field_name, parent):
        """Create translation serializer dynamically.

        Takes translatable model class (shared_model) from parent serializer and it
        may create a serializer class on the fly if no custom class was specified.
        """
        super(TranslatedFieldsField, self).bind(field_name, parent)
        # Expect 1-on-1 for now.
        related_name = field_name

        # This could all be done in __init__(), but by moving the code here,
        # it's possible to auto-detect the parent model.
        if self.shared_model is not None and self.serializer_class is not None:
            return

        # Fill in the blanks
        if self.serializer_class is None:
            # Auto detect parent model
            if self.shared_model is None:
                self.shared_model = parent.opts.model

            # Create serializer based on shared model.
            translated_model = self.shared_model._parler_meta[related_name]
            self.serializer_class = create_translated_fields_serializer(
                self.shared_model, related_name=related_name,
                meta={'fields': translated_model.get_translated_fields()}
            )
        else:
            self.shared_model = self.serializer_class.Meta.model

            # Don't need to have a 'language_code', it will be split up already,
            # so this should avoid redundant output.
            if 'language_code' in self.serializer_class.base_fields:
                raise ImproperlyConfigured("Serializer may not have a 'language_code' field")

    def to_representation(self, value):
        """Serialize translated fields.

        Simply iterate over available translations and, for each language,
        delegate serialization logic to the translation model serializer.
        """
        if value is None:
            return

        # Only need one serializer to create the native objects
        serializer = self.serializer_class()

        # Split into a dictionary per language
        result = serializers.OrderedDict()
        for translation in value.all():  # value = translations related manager
            result[translation.language_code] = serializer.to_representation(translation)

        return result

    def to_internal_value(self, data):
        """Deserialize data from translations fields.

        For each received language, delegate validation logic to
        the translation model serializer.
        """
        if data is None:
            return

        if not isinstance(data, dict):
            self.fail('invalid')

        result, errors = {}, {}
        for lang_code, model_fields in data.items():
            serializer = self.serializer_class(data=model_fields)
            if serializer.is_valid():
                result[lang_code] = serializer.data
            else:
                errors[lang_code] = serializer.errors

        if errors:
            raise serializers.ValidationError(errors)
        return result
<commit_msg>For nested translation serializers, use validated_data<commit_after># -*- coding: utf-8 -*-

"""Custom serializer fields for nested translations."""

from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured

from rest_framework import serializers

from parler_rest.utils import create_translated_fields_serializer

class TranslatedFieldsField(serializers.Field):

    """Exposing translated fields for a TranslatableModel in REST style."""

    default_error_messages = dict(serializers.Field.default_error_messages, **{
        'invalid': "Input is not a valid dict",
    })

    def __init__(self, *args, **kwargs):
        """Receive custom serializer class and model."""
        self.serializer_class = kwargs.pop('serializer_class', None)
        self.shared_model = kwargs.pop('shared_model', None)
        super(TranslatedFieldsField, self).__init__(*args, **kwargs)

    def bind(self, field_name, parent):
        """Create translation serializer dynamically.

        Takes translatable model class (shared_model) from parent serializer and it
        may create a serializer class on the fly if no custom class was specified.
        """
        super(TranslatedFieldsField, self).bind(field_name, parent)
        # Expect 1-on-1 for now.
        related_name = field_name

        # This could all be done in __init__(), but by moving the code here,
        # it's possible to auto-detect the parent model.
        if self.shared_model is not None and self.serializer_class is not None:
            return

        # Fill in the blanks
        if self.serializer_class is None:
            # Auto detect parent model
            if self.shared_model is None:
                self.shared_model = parent.opts.model

            # Create serializer based on shared model.
            translated_model = self.shared_model._parler_meta[related_name]
            self.serializer_class = create_translated_fields_serializer(
                self.shared_model, related_name=related_name,
                meta={'fields': translated_model.get_translated_fields()}
            )
        else:
            self.shared_model = self.serializer_class.Meta.model

            # Don't need to have a 'language_code', it will be split up already,
            # so this should avoid redundant output.
            if 'language_code' in self.serializer_class.base_fields:
                raise ImproperlyConfigured("Serializer may not have a 'language_code' field")

    def to_representation(self, value):
        """Serialize translated fields.

        Simply iterate over available translations and, for each language,
        delegate serialization logic to the translation model serializer.
        """
        if value is None:
            return

        # Only need one serializer to create the native objects
        serializer = self.serializer_class()

        # Split into a dictionary per language
        result = serializers.OrderedDict()
        for translation in value.all():  # value = translations related manager
            result[translation.language_code] = serializer.to_representation(translation)

        return result

    def to_internal_value(self, data):
        """Deserialize data from translations fields.

        For each received language, delegate validation logic to
        the translation model serializer.
        """
        if data is None:
            return

        if not isinstance(data, dict):
            self.fail('invalid')

        result, errors = {}, {}
        for lang_code, model_fields in data.items():
            serializer = self.serializer_class(data=model_fields)
            if serializer.is_valid():
                result[lang_code] = serializer.validated_data
            else:
                errors[lang_code] = serializer.errors

        if errors:
            raise serializers.ValidationError(errors)
        return result
<|endoftext|>


<commit_before>import pymongo
from . import KeyStore, Location
from .exceptions import ItemNotFoundError, InsufficientSpecificationError
from xmodule.x_module import XModuleDescriptor

class MongoKeyStore(KeyStore):
    """
    A Mongodb backed KeyStore
    """
    def __init__(self, host, db, collection, port=27017):
        self.collection = pymongo.connection.Connection(
            host=host,
            port=port
        )[db][collection]

        # Force mongo to report errors, at the expense of performance
        self.collection.safe = True

    def get_item(self, location):
        """
        Returns an XModuleDescriptor instance for the item at location

        If no object is found at that location, raises keystore.exceptions.ItemNotFoundError

        Searches for all matches of a partially specifed location, but raises an
        keystore.exceptions.InsufficientSpecificationError if more
        than a single object matches the query.

        location: Something that can be passed to Location
        """
        query = dict(
            ('location.{key}'.format(key=key), val)
            for (key, val)
            in Location(location).dict().items()
            if val is not None
        )
        items = self.collection.find(
            query,
            sort=[('revision', pymongo.ASCENDING)],
            limit=1,
        )
        if items.count() > 1:
            raise InsufficientSpecificationError(location)

        if items.count() == 0:
            raise ItemNotFoundError(location)

        return XModuleDescriptor.load_from_json(items[0], self.get_item)

    def create_item(self, location, editor):
        """
        Create an empty item at the specified location with the supplied editor

        location: Something that can be passed to Location
        """
        self.collection.insert({
            'location': Location(location).dict(),
            'editor': editor
        })

    def update_item(self, location, data):
        """
        Set the data in the item specified by the location to
        data

        location: Something that can be passed to Location
        data: A nested dictionary of problem data
        """
        self.collection.update(
            {'location': Location(location).dict()},
            {'$set': {'data': data}}
        )

    def update_children(self, location, children):
        """
        Set the children for the item specified by the location to
        data

        location: Something that can be passed to Location
        children: A list of child item identifiers
        """
        self.collection.update(
            {'location': Location(location).dict()},
            {'$set': {'children': children}}
        )
<commit_msg>Add documentation of mongo query syntax usage<commit_after>import pymongo
from . import KeyStore, Location
from .exceptions import ItemNotFoundError, InsufficientSpecificationError
from xmodule.x_module import XModuleDescriptor

class MongoKeyStore(KeyStore):
    """
    A Mongodb backed KeyStore
    """
    def __init__(self, host, db, collection, port=27017):
        self.collection = pymongo.connection.Connection(
            host=host,
            port=port
        )[db][collection]

        # Force mongo to report errors, at the expense of performance
        self.collection.safe = True

    def get_item(self, location):
        """
        Returns an XModuleDescriptor instance for the item at location

        If no object is found at that location, raises keystore.exceptions.ItemNotFoundError

        Searches for all matches of a partially specifed location, but raises an
        keystore.exceptions.InsufficientSpecificationError if more
        than a single object matches the query.

        location: Something that can be passed to Location
        """
        query = dict(
            ('location.{key}'.format(key=key), val)
            for (key, val)
            in Location(location).dict().items()
            if val is not None
        )
        items = self.collection.find(
            query,
            sort=[('revision', pymongo.ASCENDING)],
            limit=1,
        )
        if items.count() > 1:
            raise InsufficientSpecificationError(location)

        if items.count() == 0:
            raise ItemNotFoundError(location)

        return XModuleDescriptor.load_from_json(items[0], self.get_item)

    def create_item(self, location, editor):
        """
        Create an empty item at the specified location with the supplied editor

        location: Something that can be passed to Location
        """
        self.collection.insert({
            'location': Location(location).dict(),
            'editor': editor
        })

    def update_item(self, location, data):
        """
        Set the data in the item specified by the location to
        data

        location: Something that can be passed to Location
        data: A nested dictionary of problem data
        """

        # See http://www.mongodb.org/display/DOCS/Updating for
        # atomic update syntax
        self.collection.update(
            {'location': Location(location).dict()},
            {'$set': {'data': data}}
        )

    def update_children(self, location, children):
        """
        Set the children for the item specified by the location to
        data

        location: Something that can be passed to Location
        children: A list of child item identifiers
        """

        # See http://www.mongodb.org/display/DOCS/Updating for
        # atomic update syntax
        self.collection.update(
            {'location': Location(location).dict()},
            {'$set': {'children': children}}
        )
<|endoftext|>


<commit_before>#:coding=utf-8:
import os

from django.http import HttpRequest
from django.test import TestCase as DjangoTestCase
from django.conf import settings

from commons.views.decorators import render_to

@render_to("view_tests/render_to.html")
def myview(request):
    return {"my_value": "MY VALUE"}

class RenderToTestCase(DjangoTestCase):
    def setUp(self):
        self.template_dirs = settings.TEMPLATE_DIRS

    def tearDown(self):
        settings.TEMPLATE_DIRS = self.template_dirs

    def test_render_to(self):
        settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)
        resp = myview(HttpRequest())
        self.assertEquals(resp.content, 'MY VALUE\n')
<commit_msg>ajax_requestのテストを追加<commit_after>#:coding=utf-8:
import os

from django.http import HttpRequest
from django.test import TestCase as DjangoTestCase
from django.conf import settings

from commons.views.decorators import *

@render_to("view_tests/render_to.html")
def myview(request):
    return {"my_value": "MY VALUE"}

class RenderToTestCase(DjangoTestCase):
    def setUp(self):
        self.template_dirs = settings.TEMPLATE_DIRS

    def tearDown(self):
        settings.TEMPLATE_DIRS = self.template_dirs

    def test_render_to(self):
        settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)
        resp = myview(HttpRequest())
        self.assertEquals(resp.content, 'MY VALUE\n')

@ajax_request
def my_ajax_view(request):
    return {'my_value': 'MY VALUE'}

class AjaxResponseTestCase(DjangoTestCase):
    def test_ajax_view(self):
        resp = my_ajax_view(HttpRequest())
        self.assertEquals(resp.content, '{"my_value": "MY VALUE"}')
<|endoftext|>


<commit_before><commit_msg>Fix first time run<commit_after><|endoftext|>


<commit_before><commit_msg>[CHG] show also coresidency name in address<commit_after><|endoftext|>


<commit_before>from __future__ import print_function

import traceback
import sys
import io

from contextlib import contextmanager
from compliance_checker.suite import CheckSuite

# Py 3.4+ has contextlib.redirect_stdout to redirect stdout to a different
# stream, but use this decorated function in order to redirect output in
# previous versions
@contextmanager
def stdout_redirector(stream):
    old_stdout = sys.stdout
    sys.stdout = stream
    try:
        yield
    finally:
        sys.stdout = old_stdout

class ComplianceChecker(object):
    """
    Compliance Checker runner class.

    Ties together the entire compliance checker framework, is used from
    the command line or can be used via import.
    """
    @classmethod
    def run_checker(cls, ds_loc, checker_names, verbose, criteria, output_filename='stdout', output_format='stdout'):
        """
        Static check runner.

        @param  ds_loc          Dataset location (url or file)
        @param  checker_names    List of string names to run, should match keys of checkers dict (empty list means run all)
        @param  verbose         Verbosity of the output (0, 1, 2)
        @param  criteria        Determines failure (lenient, normal, strict)
        @param  output_filename Path to the file for output
        @param  output_format   Format of the output

        @returns                If the tests failed (based on the criteria)
        """
        cs = CheckSuite()
        ds = cs.load_dataset(ds_loc)
        score_groups = cs.run(ds, *checker_names)

        if criteria == 'normal':
            limit = 2
        elif criteria == 'strict':
            limit = 1
        elif criteria == 'lenient':
            limit = 3

        if output_format == 'text':
            if output_filename == '-':
                groups = cls.stdout_output(cs, score_groups, verbose, limit)
            # need to redirect output from stdout since print functions are
            # presently used to generate the standard report output
            else:
                with io.open(output_filename, 'w', encoding='utf-8') as f:
                    with stdout_redirector(f):
                        groups = cls.stdout_output(cs, score_groups, verbose,
                                                   limit)

        elif output_format == 'html':
            groups = cls.html_output(cs, score_groups, output_filename, ds_loc, limit)

        elif output_format == 'json':
            groups = cls.json_output(cs, score_groups, output_filename, ds_loc, limit)

        else:
            raise TypeError('Invalid format %s' % output_format)

        errors_occurred = cls.check_errors(score_groups, verbose)

        return cs.passtree(groups, limit), errors_occurred

    @classmethod
    def stdout_output(cls, cs, score_groups, verbose, limit):
        '''
        Calls output routine to display results in terminal, including scoring.
        Goes to verbose function if called by user.

        @param cs           Compliance Checker Suite
        @param score_groups List of results
        @param verbose      Integer value for verbosity level
        @param limit        The degree of strictness, 1 being the strictest, and going up from there.
        '''

        for checker, rpair in score_groups.items():
            groups, errors = rpair
            score_list, points, out_of = cs.standard_output(limit, checker, groups)
            if not verbose:
                cs.non_verbose_output_generation(score_list, groups, limit, points, out_of)
            else:
                cs.verbose_output_generation(groups, limit, points, out_of)
        return groups

    @classmethod
    def html_output(cls, cs, score_groups, output_filename, ds_loc, limit):
        '''
        Generates rendered HTML output for the compliance score(s)
        @param cs              Compliance Checker Suite
        @param score_groups    List of results
        @param output_filename The file path to output to
        @param ds_loc          Location of the source dataset
        @param limit           The degree of strictness, 1 being the strictest, and going up from there.
        '''
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if output_filename == '-':
                f = io.StringIO()
                cs.html_output(checker, groups, f, ds_loc, limit)
                f.seek(0)
                print(f.read())
            else:
                with io.open(output_filename, 'w', encoding='utf8') as f:
                    cs.html_output(checker, groups, f, ds_loc, limit)

        return groups

    @classmethod
    def json_output(cls, cs, score_groups, output_filename, ds_loc, limit):
        '''
        Generates JSON output for the ocmpliance score(s)
        @param cs              Compliance Checker Suite
        @param score_groups    List of results
        @param output_filename The file path to output to
        @param ds_loc          Location of the source dataset
        @param limit           The degree of strictness, 1 being the strictest, and going up from there.
        '''
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if output_filename == '-':
                f = io.StringIO()
                cs.json_output(checker, groups, f, ds_loc, limit)
                f.seek(0)
                print(f.read())
            else:
                with io.open(output_filename, 'w', encoding='utf8') as f:
                    cs.json_output(checker, groups, f, ds_loc, limit)

        return groups

    @classmethod
    def check_errors(cls, score_groups, verbose):
        '''
        Reports any errors (exceptions) that occurred during checking to stderr.
        Goes to verbose function if called by user.

        @param score_groups List of results
        @param verbose      Integer value for verbosity level
        '''
        errors_occurred = False
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if len(errors):
                errors_occurred = True
                print("WARNING: The following exceptions occured during the %s checker (possibly indicate compliance checker issues):" % checker, file=sys.stderr)
                for check_name, epair in errors.items():
                    print("%s.%s: %s" % (checker, check_name, epair[0]), file=sys.stderr)

                    if verbose > 0:
                        traceback.print_tb(epair[1].tb_next.tb_next)    # skip first two as they are noise from the running itself @TODO search for check_name
                        print(file=sys.stderr)

        return errors_occurred
<commit_msg>Eliminate bad default args in `run_checker`<commit_after>from __future__ import print_function

import traceback
import sys
import io

from contextlib import contextmanager
from compliance_checker.suite import CheckSuite

# Py 3.4+ has contextlib.redirect_stdout to redirect stdout to a different
# stream, but use this decorated function in order to redirect output in
# previous versions
@contextmanager
def stdout_redirector(stream):
    old_stdout = sys.stdout
    sys.stdout = stream
    try:
        yield
    finally:
        sys.stdout = old_stdout

class ComplianceChecker(object):
    """
    Compliance Checker runner class.

    Ties together the entire compliance checker framework, is used from
    the command line or can be used via import.
    """
    @classmethod
    def run_checker(cls, ds_loc, checker_names, verbose, criteria,
                    output_filename='-', output_format='text'):
        """
        Static check runner.

        @param  ds_loc          Dataset location (url or file)
        @param  checker_names    List of string names to run, should match keys of checkers dict (empty list means run all)
        @param  verbose         Verbosity of the output (0, 1, 2)
        @param  criteria        Determines failure (lenient, normal, strict)
        @param  output_filename Path to the file for output
        @param  output_format   Format of the output

        @returns                If the tests failed (based on the criteria)
        """
        cs = CheckSuite()
        ds = cs.load_dataset(ds_loc)
        score_groups = cs.run(ds, *checker_names)

        if criteria == 'normal':
            limit = 2
        elif criteria == 'strict':
            limit = 1
        elif criteria == 'lenient':
            limit = 3

        if output_format == 'text':
            if output_filename == '-':
                groups = cls.stdout_output(cs, score_groups, verbose, limit)
            # need to redirect output from stdout since print functions are
            # presently used to generate the standard report output
            else:
                with io.open(output_filename, 'w', encoding='utf-8') as f:
                    with stdout_redirector(f):
                        groups = cls.stdout_output(cs, score_groups, verbose,
                                                   limit)

        elif output_format == 'html':
            groups = cls.html_output(cs, score_groups, output_filename, ds_loc, limit)

        elif output_format == 'json':
            groups = cls.json_output(cs, score_groups, output_filename, ds_loc, limit)

        else:
            raise TypeError('Invalid format %s' % output_format)

        errors_occurred = cls.check_errors(score_groups, verbose)

        return cs.passtree(groups, limit), errors_occurred

    @classmethod
    def stdout_output(cls, cs, score_groups, verbose, limit):
        '''
        Calls output routine to display results in terminal, including scoring.
        Goes to verbose function if called by user.

        @param cs           Compliance Checker Suite
        @param score_groups List of results
        @param verbose      Integer value for verbosity level
        @param limit        The degree of strictness, 1 being the strictest, and going up from there.
        '''

        for checker, rpair in score_groups.items():
            groups, errors = rpair
            score_list, points, out_of = cs.standard_output(limit, checker, groups)
            if not verbose:
                cs.non_verbose_output_generation(score_list, groups, limit, points, out_of)
            else:
                cs.verbose_output_generation(groups, limit, points, out_of)
        return groups

    @classmethod
    def html_output(cls, cs, score_groups, output_filename, ds_loc, limit):
        '''
        Generates rendered HTML output for the compliance score(s)
        @param cs              Compliance Checker Suite
        @param score_groups    List of results
        @param output_filename The file path to output to
        @param ds_loc          Location of the source dataset
        @param limit           The degree of strictness, 1 being the strictest, and going up from there.
        '''
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if output_filename == '-':
                f = io.StringIO()
                cs.html_output(checker, groups, f, ds_loc, limit)
                f.seek(0)
                print(f.read())
            else:
                with io.open(output_filename, 'w', encoding='utf8') as f:
                    cs.html_output(checker, groups, f, ds_loc, limit)

        return groups

    @classmethod
    def json_output(cls, cs, score_groups, output_filename, ds_loc, limit):
        '''
        Generates JSON output for the ocmpliance score(s)
        @param cs              Compliance Checker Suite
        @param score_groups    List of results
        @param output_filename The file path to output to
        @param ds_loc          Location of the source dataset
        @param limit           The degree of strictness, 1 being the strictest, and going up from there.
        '''
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if output_filename == '-':
                f = io.StringIO()
                cs.json_output(checker, groups, f, ds_loc, limit)
                f.seek(0)
                print(f.read())
            else:
                with io.open(output_filename, 'w', encoding='utf8') as f:
                    cs.json_output(checker, groups, f, ds_loc, limit)

        return groups

    @classmethod
    def check_errors(cls, score_groups, verbose):
        '''
        Reports any errors (exceptions) that occurred during checking to stderr.
        Goes to verbose function if called by user.

        @param score_groups List of results
        @param verbose      Integer value for verbosity level
        '''
        errors_occurred = False
        for checker, rpair in score_groups.items():
            groups, errors = rpair
            if len(errors):
                errors_occurred = True
                print("WARNING: The following exceptions occured during the %s checker (possibly indicate compliance checker issues):" % checker, file=sys.stderr)
                for check_name, epair in errors.items():
                    print("%s.%s: %s" % (checker, check_name, epair[0]), file=sys.stderr)

                    if verbose > 0:
                        traceback.print_tb(epair[1].tb_next.tb_next)    # skip first two as they are noise from the running itself @TODO search for check_name
                        print(file=sys.stderr)

        return errors_occurred
<|endoftext|>


<commit_before># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/

import conn

def get_collections():
    """Get a list of all collections in the server"""
    path = "document/collections"
    return conn._get_paged_json(path)

def get_collection(slug):
    """Get the documents (recordings) in a collection.
    Arguments:
      slug: the name of the collection
    """
    path = "document/%s" % slug
    return conn._dunya_query_json(path)

def document(recordingid):
    """Get the available source filetypes for a Musicbrainz recording
    Arguments:
      recordingid: Musicbrainz recording ID
    Returns:
      a list of filetypes in the database for this recording"""
    path = "document/by-id/%s" % recordingid
    recording = conn._dunya_query_json(path)
    return recording

def file_for_document(recordingid, thetype, subtype=None, part=None, version=None):
    """Get the most recent derived file given a filetype
    Arguments:
      recordingid: Musicbrainz recording ID
      derivedtype: the computed filetype (from derived_filetypes_for_recording)
    Returns:
      The contents of the most recent version of the derived file"""
    path = "document/by-id/%s/%s" % (recordingid, thetype)
    args = {}
    if subtype:
        args["subtype"] = subtype
    if part:
        args["part"] = part
    if version:
        args["version"] = version
    return conn._dunya_query_file(path, **args)

def get_mp3(recordingid):
    return file_for_document(recordingid, "mp3")

<commit_msg>version parameter is v, not version<commit_after># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/

import conn

def get_collections():
    """Get a list of all collections in the server"""
    path = "document/collections"
    return conn._get_paged_json(path)

def get_collection(slug):
    """Get the documents (recordings) in a collection.
    Arguments:
      slug: the name of the collection
    """
    path = "document/%s" % slug
    return conn._dunya_query_json(path)

def document(recordingid):
    """Get the available source filetypes for a Musicbrainz recording
    Arguments:
      recordingid: Musicbrainz recording ID
    Returns:
      a list of filetypes in the database for this recording"""
    path = "document/by-id/%s" % recordingid
    recording = conn._dunya_query_json(path)
    return recording

def file_for_document(recordingid, thetype, subtype=None, part=None, version=None):
    """Get the most recent derived file given a filetype
    Arguments:
      recordingid: Musicbrainz recording ID
      derivedtype: the computed filetype (from derived_filetypes_for_recording)
    Returns:
      The contents of the most recent version of the derived file"""
    path = "document/by-id/%s/%s" % (recordingid, thetype)
    args = {}
    if subtype:
        args["subtype"] = subtype
    if part:
        args["part"] = part
    if version:
        args["v"] = version
    return conn._dunya_query_file(path, **args)

def get_mp3(recordingid):
    return file_for_document(recordingid, "mp3")

<|endoftext|>


<commit_before>from __future__ import division
import math

import numpy

from chainer.backends import cuda
from chainer import optimizer

_default_hyperparam = optimizer.Hyperparameter()
_default_hyperparam.alpha = 0.001
_default_hyperparam.beta1 = 0.9
_default_hyperparam.beta2 = 0.999
_default_hyperparam.eps = 1e-8
_default_hyperparam.eta = 1.0
_default_hyperparam.weight_decay_rate = 0
_default_hyperparam.amsgrad = False

def _learning_rate(hp, t):
    if t == 0:
        raise RuntimeError(
            'Can\'t determine the learning rate of Adam optimizer '
            'because the update steps have not been started.')
    fix1 = 1. - math.pow(hp.beta1, t)
    fix2 = 1. - math.pow(hp.beta2, t)
    return hp.alpha * math.sqrt(fix2) / fix1

class AdamRule(optimizer.UpdateRule):

    """Update rule of Adam optimization algorithm.

    See: `Adam: A Method for Stochastic Optimization \
          <https://arxiv.org/abs/1412.6980v8>`_

    Modified for proper weight decay.

    See: `Fixing Weight Decay Regularization in Adam \
          <https://openreview.net/forum?id=rk6qdGgCZ>`_

    With option to use AMSGrad variant of Adam.

    See: `On the Convergence of Adam and Beyond \
          <https://openreview.net/forum?id=ryQu7f-RZ>`_

    See :class:`~chainer.optimizers.Adam` for the default values
    of the hyperparameters.

    Args:
        parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter
            that provides the default values.
        alpha (float): Coefficient of learning rate.
        beta1 (float): Exponential decay rate of the first order moment.
        beta2 (float): Exponential decay rate of the second order moment.
        eps (float): Small value for the numerical stability.
        eta (float): Schedule multiplier, can be used for warm restarts.
        weight_decay_rate (float): Weight decay rate.
        amsgrad (bool): Whether to use the AMSGrad variant of Adam.

    """

    def __init__(self, parent_hyperparam=None,
                 alpha=None, beta1=None, beta2=None, eps=None,
                 eta=None, weight_decay_rate=None, amsgrad=None):
        super(AdamRule, self).__init__(
            parent_hyperparam or _default_hyperparam)
        if alpha is not None:
            self.hyperparam.alpha = alpha
        if beta1 is not None:
            self.hyperparam.beta1 = beta1
        if beta2 is not None:
            self.hyperparam.beta2 = beta2
        if eps is not None:
            self.hyperparam.eps = eps
        if eta is not None:
            self.hyperparam.eta = eta
        if weight_decay_rate is not None:
            self.hyperparam.weight_decay_rate = weight_decay_rate
        if amsgrad is not None:
            self.hyperparam.amsgrad = amsgrad

    def init_state(self, param):
        xp = cuda.get_array_module(param.data)
        with cuda.get_device_from_array(param.data):
            self.state['m'] = xp.zeros_like(param.data)
            self.state['v'] = xp.zeros_like(param.data)
            if self.hyperparam.amsgrad:
                self.state['vhat'] = xp.zeros_like(param.data)

    def update_core_cpu(self, param):
        grad = param.grad
        if grad is None:
            return
        hp = self.hyperparam
        eps = grad.dtype.type(hp.eps)
        if hp.eps != 0 and eps == 0:
            raise ValueError(
                'eps of Adam optimizer is too small for {} ({})'.format(
                    grad.dtype.name, hp.eps))
        m, v = self.state['m'], self.state['v']

        m += (1 - hp.beta1) * (grad - m)
        v += (1 - hp.beta2) * (grad * grad - v)

        if hp.amsgrad:
            vhat = self.state['vhat']
            numpy.maximum(vhat, v, out=vhat)
        else:
            vhat = v
        param.data -= hp.eta * (self.lr * m / (numpy.sqrt(vhat) + hp.eps) +
                                hp.weight_decay_rate * param.data)

    def update_core_gpu(self, param):
        grad = param.grad
        if grad is None:
            return

        hp = self.hyperparam
        eps = grad.dtype.type(hp.eps)
        if hp.eps != 0 and eps == 0:
            raise ValueError(
                'eps of Adam optimizer is too small for {} ({})'.format(
                    grad.dtype.name, hp.eps))
        if hp.amsgrad:
            cuda.elementwise(
                'T grad, T lr, T one_minus_beta1, T one_minus_beta2, T eps, \
                 T eta, T weight_decay_rate',
                'T param, T m, T v, T vhat',
                '''m += one_minus_beta1 * (grad - m);
                   v += one_minus_beta2 * (grad * grad - v);
                   vhat = max(vhat, v);
                   param -= eta * (lr * m / (sqrt(vhat) + eps) +
                                   weight_decay_rate * param);''',
                'adam')(grad, self.lr, 1 - hp.beta1,
                        1 - hp.beta2, hp.eps,
                        hp.eta, hp.weight_decay_rate,
                        param.data, self.state['m'], self.state['v'],
                        self.state['vhat'])
        else:
            cuda.elementwise(
                'T grad, T lr, T one_minus_beta1, T one_minus_beta2, T eps, \
                 T eta, T weight_decay_rate',
                'T param, T m, T v',
                '''m += one_minus_beta1 * (grad - m);
                   v += one_minus_beta2 * (grad * grad - v);
                   param -= eta * (lr * m / (sqrt(v) + eps) +
                                   weight_decay_rate * param);''',
                'adam')(grad, self.lr, 1 - hp.beta1,
                        1 - hp.beta2, hp.eps,
                        hp.eta, hp.weight_decay_rate,
                        param.data, self.state['m'], self.state['v'])

    @property
    def lr(self):
        return _learning_rate(self.hyperparam, self.t)

class Adam(optimizer.GradientMethod):

    """Adam optimizer.

    See: `Adam: A Method for Stochastic Optimization \
          <https://arxiv.org/abs/1412.6980v8>`_

    Modified for proper weight decay (also called AdamW).
    AdamW introduces the additional parameters ``eta``
    and ``weight_decay_rate``, which can be used to properly scale the
    learning rate, and decouple the weight decay rate from ``alpha``,
    as shown in the below paper.

    Note that with the default values ``eta = 1`` and
    ``weight_decay_rate = 0``, this implementation is identical to
    the standard Adam method.

    See: `Fixing Weight Decay Regularization in Adam \
          <https://openreview.net/forum?id=rk6qdGgCZ>`_

    A flag ``amsgrad`` to use the AMSGrad variant of Adam from
    the paper: `On the Convergence of Adam and Beyond \
               <https://openreview.net/forum?id=ryQu7f-RZ>`_

    Args:
        alpha (float): Coefficient of learning rate.
        beta1 (float): Exponential decay rate of the first order moment.
        beta2 (float): Exponential decay rate of the second order moment.
        eps (float): Small value for the numerical stability.
        eta (float): Schedule multiplier, can be used for warm restarts.
        weight_decay_rate (float): Weight decay rate.
        amsgrad (bool): Whether to use AMSGrad variant of Adam.

    """

    def __init__(self,
                 alpha=_default_hyperparam.alpha,
                 beta1=_default_hyperparam.beta1,
                 beta2=_default_hyperparam.beta2,
                 eps=_default_hyperparam.eps,
                 eta=_default_hyperparam.eta,
                 weight_decay_rate=_default_hyperparam.weight_decay_rate,
                 amsgrad=_default_hyperparam.amsgrad):
        super(Adam, self).__init__()
        self.hyperparam.alpha = alpha
        self.hyperparam.beta1 = beta1
        self.hyperparam.beta2 = beta2
        self.hyperparam.eps = eps
        self.hyperparam.eta = eta
        self.hyperparam.weight_decay_rate = weight_decay_rate
        self.hyperparam.amsgrad = amsgrad

    alpha = optimizer.HyperparameterProxy('alpha')
    beta1 = optimizer.HyperparameterProxy('beta1')
    beta2 = optimizer.HyperparameterProxy('beta2')
    eps = optimizer.HyperparameterProxy('eps')
    eta = optimizer.HyperparameterProxy('eta')
    weight_decay_rate = optimizer.HyperparameterProxy('weight_decay_rate')
    amsgrad = optimizer.HyperparameterProxy('amsgrad')

    def create_update_rule(self):
        return AdamRule(self.hyperparam)

    @property
    def lr(self):
        return _learning_rate(self.hyperparam, self.t)
<commit_msg>adam ideep modified by recomendation [provided by Ikei]<commit_after>from __future__ import division
import math

import numpy

from chainer.backends import cuda
from chainer.backends import intel64
from chainer import optimizer

_default_hyperparam = optimizer.Hyperparameter()
_default_hyperparam.alpha = 0.001
_default_hyperparam.beta1 = 0.9
_default_hyperparam.beta2 = 0.999
_default_hyperparam.eps = 1e-8
_default_hyperparam.eta = 1.0
_default_hyperparam.weight_decay_rate = 0
_default_hyperparam.amsgrad = False

def _learning_rate(hp, t):
    if t == 0:
        raise RuntimeError(
            'Can\'t determine the learning rate of Adam optimizer '
            'because the update steps have not been started.')
    fix1 = 1. - math.pow(hp.beta1, t)
    fix2 = 1. - math.pow(hp.beta2, t)
    return hp.alpha * math.sqrt(fix2) / fix1

class AdamRule(optimizer.UpdateRule):

    """Update rule of Adam optimization algorithm.

    See: `Adam: A Method for Stochastic Optimization \
          <https://arxiv.org/abs/1412.6980v8>`_

    Modified for proper weight decay.

    See: `Fixing Weight Decay Regularization in Adam \
          <https://openreview.net/forum?id=rk6qdGgCZ>`_

    With option to use AMSGrad variant of Adam.

    See: `On the Convergence of Adam and Beyond \
          <https://openreview.net/forum?id=ryQu7f-RZ>`_

    See :class:`~chainer.optimizers.Adam` for the default values
    of the hyperparameters.

    Args:
        parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter
            that provides the default values.
        alpha (float): Coefficient of learning rate.
        beta1 (float): Exponential decay rate of the first order moment.
        beta2 (float): Exponential decay rate of the second order moment.
        eps (float): Small value for the numerical stability.
        eta (float): Schedule multiplier, can be used for warm restarts.
        weight_decay_rate (float): Weight decay rate.
        amsgrad (bool): Whether to use the AMSGrad variant of Adam.

    """

    def __init__(self, parent_hyperparam=None,
                 alpha=None, beta1=None, beta2=None, eps=None,
                 eta=None, weight_decay_rate=None, amsgrad=None):
        super(AdamRule, self).__init__(
            parent_hyperparam or _default_hyperparam)
        if alpha is not None:
            self.hyperparam.alpha = alpha
        if beta1 is not None:
            self.hyperparam.beta1 = beta1
        if beta2 is not None:
            self.hyperparam.beta2 = beta2
        if eps is not None:
            self.hyperparam.eps = eps
        if eta is not None:
            self.hyperparam.eta = eta
        if weight_decay_rate is not None:
            self.hyperparam.weight_decay_rate = weight_decay_rate
        if amsgrad is not None:
            self.hyperparam.amsgrad = amsgrad

    def init_state(self, param):
        xp = cuda.get_array_module(param.data)
        with cuda.get_device_from_array(param.data):
            self.state['m'] = xp.zeros_like(param.data)
            self.state['v'] = xp.zeros_like(param.data)
            if self.hyperparam.amsgrad:
                self.state['vhat'] = xp.zeros_like(param.data)

        # For iDeep
        if (isinstance(param.data, intel64.mdarray)
                and intel64.inputs_all_ready((self.state['m'],))
                and intel64.inputs_all_ready((self.state['v'],))):
            self.state['m'] = intel64.ideep.array(
                self.state['m'], itype=intel64.ideep.wgt_array)
            self.state['v'] = intel64.ideep.array(
                self.state['v'], itype=intel64.ideep.wgt_array)

    def update_core_cpu(self, param):
        grad = param.grad
        if grad is None:
            return
        hp = self.hyperparam
        eps = grad.dtype.type(hp.eps)
        if hp.eps != 0 and eps == 0:
            raise ValueError(
                'eps of Adam optimizer is too small for {} ({})'.format(
                    grad.dtype.name, hp.eps))
        m, v = self.state['m'], self.state['v']
        if (isinstance(m, intel64.mdarray) and
            isinstance(v, intel64.mdarray)):
            m.inplace_axpby(1.0, 1.0 - hp.beta1, grad - m)
            v.inplace_axpby(1.0, 1.0 - hp.beta2, grad*grad - v)
            if hp.amsgrad:
                vhat = self.state['vhat']
                numpy.maximum(vhat, v, out=vhat)
                param.data.inplace_axpby(1.0 - hp.weight_decay_rate, -hp.eta,
                                     self.lr * m / (numpy.sqrt(vhat) + hp.eps))
            else:
                param.data.inplace_axpby(1.0 - hp.weight_decay_rate, -hp.eta,
                                     self.lr * m / (numpy.sqrt(v) + hp.eps))
        else:
            m += (1 - hp.beta1) * (grad - m)
            v += (1 - hp.beta2) * (grad * grad - v)
            if hp.amsgrad:
                vhat = self.state['vhat']
                numpy.maximum(vhat, v, out=vhat)
            else:
                vhat = v
            param.data -= hp.eta * (self.lr * m / (numpy.sqrt(vhat) + hp.eps) +
                                hp.weight_decay_rate * param.data)

    def update_core_gpu(self, param):
        grad = param.grad
        if grad is None:
            return

        hp = self.hyperparam
        eps = grad.dtype.type(hp.eps)
        if hp.eps != 0 and eps == 0:
            raise ValueError(
                'eps of Adam optimizer is too small for {} ({})'.format(
                    grad.dtype.name, hp.eps))
        if hp.amsgrad:
            cuda.elementwise(
                'T grad, T lr, T one_minus_beta1, T one_minus_beta2, T eps, \
                 T eta, T weight_decay_rate',
                'T param, T m, T v, T vhat',
                '''m += one_minus_beta1 * (grad - m);
                   v += one_minus_beta2 * (grad * grad - v);
                   vhat = max(vhat, v);
                   param -= eta * (lr * m / (sqrt(vhat) + eps) +
                                   weight_decay_rate * param);''',
                'adam')(grad, self.lr, 1 - hp.beta1,
                        1 - hp.beta2, hp.eps,
                        hp.eta, hp.weight_decay_rate,
                        param.data, self.state['m'], self.state['v'],
                        self.state['vhat'])
        else:
            cuda.elementwise(
                'T grad, T lr, T one_minus_beta1, T one_minus_beta2, T eps, \
                 T eta, T weight_decay_rate',
                'T param, T m, T v',
                '''m += one_minus_beta1 * (grad - m);
                   v += one_minus_beta2 * (grad * grad - v);
                   param -= eta * (lr * m / (sqrt(v) + eps) +
                                   weight_decay_rate * param);''',
                'adam')(grad, self.lr, 1 - hp.beta1,
                        1 - hp.beta2, hp.eps,
                        hp.eta, hp.weight_decay_rate,
                        param.data, self.state['m'], self.state['v'])

    @property
    def lr(self):
        return _learning_rate(self.hyperparam, self.t)

class Adam(optimizer.GradientMethod):

    """Adam optimizer.

    See: `Adam: A Method for Stochastic Optimization \
          <https://arxiv.org/abs/1412.6980v8>`_

    Modified for proper weight decay (also called AdamW).
    AdamW introduces the additional parameters ``eta``
    and ``weight_decay_rate``, which can be used to properly scale the
    learning rate, and decouple the weight decay rate from ``alpha``,
    as shown in the below paper.

    Note that with the default values ``eta = 1`` and
    ``weight_decay_rate = 0``, this implementation is identical to
    the standard Adam method.

    See: `Fixing Weight Decay Regularization in Adam \
          <https://openreview.net/forum?id=rk6qdGgCZ>`_

    A flag ``amsgrad`` to use the AMSGrad variant of Adam from
    the paper: `On the Convergence of Adam and Beyond \
               <https://openreview.net/forum?id=ryQu7f-RZ>`_

    Args:
        alpha (float): Coefficient of learning rate.
        beta1 (float): Exponential decay rate of the first order moment.
        beta2 (float): Exponential decay rate of the second order moment.
        eps (float): Small value for the numerical stability.
        eta (float): Schedule multiplier, can be used for warm restarts.
        weight_decay_rate (float): Weight decay rate.
        amsgrad (bool): Whether to use AMSGrad variant of Adam.

    """

    def __init__(self,
                 alpha=_default_hyperparam.alpha,
                 beta1=_default_hyperparam.beta1,
                 beta2=_default_hyperparam.beta2,
                 eps=_default_hyperparam.eps,
                 eta=_default_hyperparam.eta,
                 weight_decay_rate=_default_hyperparam.weight_decay_rate,
                 amsgrad=_default_hyperparam.amsgrad):
        super(Adam, self).__init__()
        self.hyperparam.alpha = alpha
        self.hyperparam.beta1 = beta1
        self.hyperparam.beta2 = beta2
        self.hyperparam.eps = eps
        self.hyperparam.eta = eta
        self.hyperparam.weight_decay_rate = weight_decay_rate
        self.hyperparam.amsgrad = amsgrad

    alpha = optimizer.HyperparameterProxy('alpha')
    beta1 = optimizer.HyperparameterProxy('beta1')
    beta2 = optimizer.HyperparameterProxy('beta2')
    eps = optimizer.HyperparameterProxy('eps')
    eta = optimizer.HyperparameterProxy('eta')
    weight_decay_rate = optimizer.HyperparameterProxy('weight_decay_rate')
    amsgrad = optimizer.HyperparameterProxy('amsgrad')

    def create_update_rule(self):
        return AdamRule(self.hyperparam)

    @property
    def lr(self):
        return _learning_rate(self.hyperparam, self.t)
<|endoftext|>


<commit_before>import chainerx
from chainerx import _docs

def set_docs():
    _docs.set_doc(
        chainerx.backward,
        """backward(outputs, *, enable_double_backprop=False)
Runs backpropagation.

On backpropagation (a.k.a. backprop),
the computational graph is traversed backward starting from the output arrays,
up until the root arrays on which :func:`ndarray.require_grad()` have been
called.

Backpropagation uses :data:`ndarray.grad <chainerx.ndarray.grad>` held by
the output arrays as the initial gradients.
You can manually assign them before calling this function.
Otherwise, they are assumed to be 1.

To enable higher order differentiation, pass ``enable_double_backprop=True``
so that you can further run backpropagation from the resulting gradient arrays.
Note that enabling it results in larger memory consumption needed to store the
gradients w.r.t intermediate arrays that are required for the second gradient
computation.

Note:
    The whole process of backpropagation is executed in C++, except those
    operations whose backward computation falls back to the corresponding
    Python implementation. Currently this function does not release the GIL at
    all.

Args:
    outputs (~chainerx.ndarray or list of ndarrays):
        Output arrays from which backpropagation starts.
    enable_double_backprop (bool): If ``True``,
        a computational trace of the whole backpropagation procedure is
        recorded to the computational graph so that one can further do
        backpropagation from the resulting gradients.

.. seealso::
    * :meth:`chainerx.ndarray.backward`
""")

    _docs.set_doc(
        chainerx.grad,
        """grad(outputs, inputs, *, enable_double_backprop=False)
Computes and returns the gradients of the outputs w.r.t. the inputs.

This function differs from :func:`chainerx.backward` in the sense that
gradients are returned instead of being added to the gradients held by the
inputs. Gradients held by the inputs are not modified. Also, instead of
traversing through the whole graph starting from the outputs, a sub-graph is
extracted for computation. This means that is is more efficient, especially
for larger computational graphs.

Args:
    outputs (list of ndarrays):
        Output arrays from which backpropagation starts.
    inputs (list of ndarrays):
        Input arrays of which this functions computes the gradients w.r.t.
    enable_double_backprop (bool): If ``True``,
        a computational trace of the whole backpropagation procedure is
        recorded to the computational graph so that one can further do
        backpropagation from the resulting gradients.

Returns:
    list of :class:`~chainerx.ndarray`\\ s:
        A list of gradients. The list always has the same length as the number
        of inputs.

.. seealso::
    * :func:`chainerx.backward`
    * :func:`chainer.grad`
""")

    _docs.set_doc(
        chainerx.no_backprop_mode,
        """no_backprop_mode()
Creates a context manager which temporarily disables backpropagation.

Within this context, no computational graph will be formed unless
:meth:`~chainerx.force_backprop_mode` is used.

Arrays resulting from operations enclosed with this context will be
disconnected from the computational graph. Trying to perform backpropagation
from such arrays would result in an error.

.. code-block:: py

    x = chainerx.array([4, 3], numpy.float32)
    x.require_grad()

    with chainerx.no_backprop_mode():
        y = 2 * x + 1

    y.backward()  # ! error

Benefits of ``no_backprop_mode`` include reduced CPU overhead of building
computational graphs, and reduced consumption of device memory that
would be otherwise retained for backward propagation.

.. seealso::
    * :func:`chainerx.force_backprop_mode`
    * :func:`chainerx.is_backprop_required`
    * :func:`chainer.no_backprop_mode`
""")

    _docs.set_doc(
        chainerx.force_backprop_mode,
        """force_backprop_mode()
Creates a context manager which temporarily enables backpropagation.

This context re-enables backpropagation that is disabled by
any surrounding :func:`~chainerx.no_backprop_mode` context.

.. code-block:: py

    x = chainerx.array([4, 3], numpy.float32)
    x.require_grad()

    with chainerx.no_backprop_mode():
        with chainerx.force_backprop_mode():
            y = 2 * x + 1

    y.backward()
    x.grad
    # array([2., 2.], shape=(2,), dtype=float32, device='native:0')

.. seealso::
    * :func:`chainerx.no_backprop_mode`
    * :func:`chainerx.is_backprop_required`
    * :func:`chainer.force_backprop_mode`
""")

    _docs.set_doc(
        chainerx.is_backprop_required,
        """is_backprop_required()
Returns whether the backpropagation is enabled in the current thread.

The result is affect by :func:`chainerx.no_backprop_mode` and
:func:`chainerx.force_backprop_mode`.

.. seealso::
    * :func:`chainerx.no_backprop_mode`
    * :func:`chainerx.force_backprop_mode`
""")
<commit_msg>Fix spelling error<commit_after>import chainerx
from chainerx import _docs

def set_docs():
    _docs.set_doc(
        chainerx.backward,
        """backward(outputs, *, enable_double_backprop=False)
Runs backpropagation.

On backpropagation (a.k.a. backprop),
the computational graph is traversed backward starting from the output arrays,
up until the root arrays on which :func:`ndarray.require_grad()` have been
called.

Backpropagation uses :data:`ndarray.grad <chainerx.ndarray.grad>` held by
the output arrays as the initial gradients.
You can manually assign them before calling this function.
Otherwise, they are assumed to be 1.

To enable higher order differentiation, pass ``enable_double_backprop=True``
so that you can further run backpropagation from the resulting gradient arrays.
Note that enabling it results in larger memory consumption needed to store the
gradients w.r.t intermediate arrays that are required for the second gradient
computation.

Note:
    The whole process of backpropagation is executed in C++, except those
    operations whose backward computation falls back to the corresponding
    Python implementation. Currently this function does not release the GIL at
    all.

Args:
    outputs (~chainerx.ndarray or list of ndarrays):
        Output arrays from which backpropagation starts.
    enable_double_backprop (bool): If ``True``,
        a computational trace of the whole backpropagation procedure is
        recorded to the computational graph so that one can further do
        backpropagation from the resulting gradients.

.. seealso::
    * :meth:`chainerx.ndarray.backward`
""")

    _docs.set_doc(
        chainerx.grad,
        """grad(outputs, inputs, *, enable_double_backprop=False)
Computes and returns the gradients of the outputs w.r.t. the inputs.

This function differs from :func:`chainerx.backward` in the sense that
gradients are returned instead of being added to the gradients held by the
inputs. Gradients held by the inputs are not modified. Also, instead of
traversing through the whole graph starting from the outputs, a sub-graph is
extracted for computation. This means that is is more efficient, especially
for larger computational graphs.

Args:
    outputs (list of ndarrays):
        Output arrays from which backpropagation starts.
    inputs (list of ndarrays):
        Input arrays of which this function computes the gradients w.r.t.
    enable_double_backprop (bool): If ``True``,
        a computational trace of the whole backpropagation procedure is
        recorded to the computational graph so that one can further do
        backpropagation from the resulting gradients.

Returns:
    list of :class:`~chainerx.ndarray`\\ s:
        A list of gradients. The list always has the same length as the number
        of inputs.

.. seealso::
    * :func:`chainerx.backward`
    * :func:`chainer.grad`
""")

    _docs.set_doc(
        chainerx.no_backprop_mode,
        """no_backprop_mode()
Creates a context manager which temporarily disables backpropagation.

Within this context, no computational graph will be formed unless
:meth:`~chainerx.force_backprop_mode` is used.

Arrays resulting from operations enclosed with this context will be
disconnected from the computational graph. Trying to perform backpropagation
from such arrays would result in an error.

.. code-block:: py

    x = chainerx.array([4, 3], numpy.float32)
    x.require_grad()

    with chainerx.no_backprop_mode():
        y = 2 * x + 1

    y.backward()  # ! error

Benefits of ``no_backprop_mode`` include reduced CPU overhead of building
computational graphs, and reduced consumption of device memory that
would be otherwise retained for backward propagation.

.. seealso::
    * :func:`chainerx.force_backprop_mode`
    * :func:`chainerx.is_backprop_required`
    * :func:`chainer.no_backprop_mode`
""")

    _docs.set_doc(
        chainerx.force_backprop_mode,
        """force_backprop_mode()
Creates a context manager which temporarily enables backpropagation.

This context re-enables backpropagation that is disabled by
any surrounding :func:`~chainerx.no_backprop_mode` context.

.. code-block:: py

    x = chainerx.array([4, 3], numpy.float32)
    x.require_grad()

    with chainerx.no_backprop_mode():
        with chainerx.force_backprop_mode():
            y = 2 * x + 1

    y.backward()
    x.grad
    # array([2., 2.], shape=(2,), dtype=float32, device='native:0')

.. seealso::
    * :func:`chainerx.no_backprop_mode`
    * :func:`chainerx.is_backprop_required`
    * :func:`chainer.force_backprop_mode`
""")

    _docs.set_doc(
        chainerx.is_backprop_required,
        """is_backprop_required()
Returns whether the backpropagation is enabled in the current thread.

The result is affect by :func:`chainerx.no_backprop_mode` and
:func:`chainerx.force_backprop_mode`.

.. seealso::
    * :func:`chainerx.no_backprop_mode`
    * :func:`chainerx.force_backprop_mode`
""")
<|endoftext|>