<commit_before>import logging, numpy, openravepy, prpy
from prpy.action import ActionMethod
from prpy.planning.base import PlanningError
from prpy.util import ComputeEnabledAABB

logger = logging.getLogger('herbpy')

@ActionMethod
def GrabBlock(robot, block, table, manip=None, preshape=[1.7, 1.7, 0.2, 2.45],
              **kw_args):
    """
    @param robot The robot performing the grasp
    @param block The block to grab
    @param table The table, or object, the block is resting on
    @param manip The manipulator to use to grab the block
      (if None active manipulator is used)
    """
    from prpy.rave import AllDisabled, Disabled
    from prpy.viz import RenderTSRList, RenderVector

    env = robot.GetEnv()

    if manip is None:
        manip = robot.GetActiveManipulator()

    # First move the hand to the right preshape
    manip.hand.MoveHand(*preshape)

    # Get a TSR to move near the block.
    block_tsr_list = robot.tsrlibrary(block, 'grasp', manip=manip)

    # Plan to a pose above the block
    with RenderTSRList(block_tsr_list, robot.GetEnv()):
        with Disabled(table, padding_only=True):
            manip.PlanToTSR(block_tsr_list)

    try:
        with AllDisabled(env, [table, block], padding_only=True):
            # Move down until touching the table
            with env:
                start_point = manip.GetEndEffectorTransform()[0:3, 3]
                table_aabb = ComputeEnabledAABB(table)
                table_height = table_aabb.pos()[2] + table_aabb.extents()[2]
                # 0.14 is the distance from finger tip to end-effector frame
                current_finger_height = manip.GetEndEffectorTransform()[2,3] - 0.14
                min_distance = current_finger_height - table_height

            down_direction = [0., 0., -1.]
            with RenderVector(start_point, down_direction, min_distance, env):
                manip.MoveUntilTouch(direction=down_direction, timelimit=5,
                    distance=min_distance, max_distance=min_distance + 0.05,
                    ignore_collisions=[block, table])

            # Move parallel to the table to funnel the block into the fingers
            # by projecting the -x direction of end-effector onto the xy-plane.
            with env:
                start_point = manip.GetEndEffectorTransform()[0:3, 3]
                funnel_direction = -1. * manip.GetEndEffectorTransform()[:3,0]
                funnel_direction[2] = 0.

            # TODO: We should only have to disable the block for this. Why does
            # this fail if we do not disable the table?
            with AllDisabled(env, [block, table]):
                with RenderVector(start_point, funnel_direction, 0.1, env):
                    manip.PlanToEndEffectorOffset(direction=funnel_direction,
                        distance=0.06, max_distance=0.1, timelimit=5.)

        # Close the finger to grab the block
        manip.hand.MoveHand(f3=1.7)

        # Compute the pose of the block in the hand frame
        with env:
            local_p = [0.01, 0, 0.24, 1.0]
            hand_pose = manip.GetEndEffectorTransform()
            world_p = numpy.dot(hand_pose, local_p)
            block_pose = block.GetTransform()
            block_pose[:,3] = world_p
            block_relative = numpy.dot(numpy.linalg.inv(hand_pose), block_pose)

        # Now lift the block up off the table
        with AllDisabled(env, [block, table]):
            manip.PlanToEndEffectorOffset(direction=[0, 0, 1], distance=0.05, timelimit=5)

        # OpenRAVE trick to hallucinate the block into the correct pose relative to the hand
        with env:
            hand_pose = manip.GetEndEffectorTransform()
            block_pose = numpy.dot(hand_pose, block_relative)
            block.SetTransform(block_pose)
            manip.GetRobot().Grab(block)

    except PlanningError as e:
        logger.error('Failed to complete block grasp')
        raise

@ActionMethod
def PlaceBlock(robot, block, on_obj, center=True, manip=None, **kw_args):
    """
    Place a block on an object
    """
    env = robot.GetEnv()

    # Get a tsr for this position
    object_place_list = robot.tsrlibrary(on_obj, 'point_on', manip=manip)
    place_tsr_list = robot.tsrlibrary(block, 'place_on',
                                      pose_tsr_chain=object_place_list[0],
                                      manip=manip)

    # Plan there
    with prpy.viz.RenderTSRList(object_place_list, robot.GetEnv()):
        manip.PlanToTSR(place_tsr_list)

    # Open the hand and drop the block
    manip.hand.MoveHand(f3=0.2)

    with env:
        manip.GetRobot().Release(block)

        # Move the block down until it hits something
        block_pose = block.GetTransform()
        while not env.CheckCollision(block) and block_pose[2,3] > 0.0:
            block_pose[2,3] -= 0.02
            block.SetTransform(block_pose)

<commit_msg>Adding execute=True flag to block action<commit_after>import logging, numpy, openravepy, prpy
from prpy.action import ActionMethod
from prpy.planning.base import PlanningError
from prpy.util import ComputeEnabledAABB

logger = logging.getLogger('herbpy')

@ActionMethod
def GrabBlock(robot, block, table, manip=None, preshape=[1.7, 1.7, 0.2, 2.45],
              **kw_args):
    """
    @param robot The robot performing the grasp
    @param block The block to grab
    @param table The table, or object, the block is resting on
    @param manip The manipulator to use to grab the block
      (if None active manipulator is used)
    """
    from prpy.rave import AllDisabled, Disabled
    from prpy.viz import RenderTSRList, RenderVector

    env = robot.GetEnv()

    if manip is None:
        manip = robot.GetActiveManipulator()

    # First move the hand to the right preshape
    manip.hand.MoveHand(*preshape)

    # Get a TSR to move near the block.
    block_tsr_list = robot.tsrlibrary(block, 'grasp', manip=manip)

    # Plan to a pose above the block
    with RenderTSRList(block_tsr_list, robot.GetEnv()):
        with Disabled(table, padding_only=True):
            manip.PlanToTSR(block_tsr_list, execute=True)

    try:
        with AllDisabled(env, [table, block], padding_only=True):
            # Move down until touching the table
            with env:
                start_point = manip.GetEndEffectorTransform()[0:3, 3]
                table_aabb = ComputeEnabledAABB(table)
                table_height = table_aabb.pos()[2] + table_aabb.extents()[2]
                # 0.14 is the distance from finger tip to end-effector frame
                current_finger_height = manip.GetEndEffectorTransform()[2,3] - 0.14
                min_distance = 0.04# current_finger_height - table_height

            down_direction = [0., 0., -1.]
            with RenderVector(start_point, down_direction, min_distance, env):
                manip.MoveUntilTouch(direction=down_direction, timelimit=5,
                    distance=min_distance, max_distance=min_distance + 0.05,
                    ignore_collisions=[block, table])

            # Move parallel to the table to funnel the block into the fingers
            # by projecting the -x direction of end-effector onto the xy-plane.
            with env:
                start_point = manip.GetEndEffectorTransform()[0:3, 3]
                funnel_direction = -1. * manip.GetEndEffectorTransform()[:3,0]
                funnel_direction[2] = 0.

            # TODO: We should only have to disable the block for this. Why does
            # this fail if we do not disable the table?
            with AllDisabled(env, [block, table]):
                with RenderVector(start_point, funnel_direction, 0.1, env):
                    manip.PlanToEndEffectorOffset(direction=funnel_direction,
                                                  distance=0.08, max_distance=0.12,
                                                  timelimit=5., execute=True)

        # Close the finger to grab the block
        manip.hand.MoveHand(f3=1.7)

        # Compute the pose of the block in the hand frame
        with env:
            local_p = [0.01, 0, 0.24, 1.0]
            hand_pose = manip.GetEndEffectorTransform()
            world_p = numpy.dot(hand_pose, local_p)
            block_pose = block.GetTransform()
            block_pose[:,3] = world_p
            block_relative = numpy.dot(numpy.linalg.inv(hand_pose), block_pose)

        # Now lift the block up off the table
        with AllDisabled(env, [block, table]):
            manip.PlanToEndEffectorOffset(direction=[0, 0, 1], distance=0.05,
                                          timelimit=5, execute=True)

        # OpenRAVE trick to hallucinate the block into the correct pose relative to the hand
        with env:
            hand_pose = manip.GetEndEffectorTransform()
            block_pose = numpy.dot(hand_pose, block_relative)
            block.SetTransform(block_pose)
            manip.GetRobot().Grab(block)

    except PlanningError as e:
        logger.error('Failed to complete block grasp')
        raise

@ActionMethod
def PlaceBlock(robot, block, on_obj, center=True, manip=None, **kw_args):
    """
    Place a block on an object
    """
    env = robot.GetEnv()

    # Get a tsr for this position
    object_place_list = robot.tsrlibrary(on_obj, 'point_on', manip=manip)
    place_tsr_list = robot.tsrlibrary(block, 'place_on',
                                      pose_tsr_chain=object_place_list[0],
                                      manip=manip)

    # Plan there
    with prpy.viz.RenderTSRList(object_place_list, robot.GetEnv()):
        manip.PlanToTSR(place_tsr_list, execute=True)

    # Open the hand and drop the block
    manip.hand.MoveHand(f3=0.2)

    with env:
        manip.GetRobot().Release(block)

        # Move the block down until it hits something
        block_pose = block.GetTransform()
        while not env.CheckCollision(block) and block_pose[2,3] > 0.0:
            block_pose[2,3] -= 0.02
            block.SetTransform(block_pose)

<|endoftext|>


<commit_before>import csv
import gzip
import sys
from django.core.management.base import BaseCommand
from xbrowse.core.variant_filters import get_default_variant_filter
from xbrowse.variant_search.family import get_variants_with_inheritance_mode, get_variants
from xbrowse_server import mall
from xbrowse_server.base.models import Project, Family
from xbrowse_server.mall import get_mall, get_reference, get_datastore
from settings import CLINVAR_VARIANTS

AB_threshold = 15
GQ_threshold = 20
DP_threshold = 10
g1k_freq_threshold = 0.01
g1k_popmax_freq_threshold = 0.01
exac_freq_threshold = 0.01
exac_popmax_threshold = 0.01
merck_wgs_3793_threshold = 0.05

def get_gene_symbol(variant):
    gene_id = variant.annotation['vep_annotation'][
        variant.annotation['worst_vep_annotation_index']]['gene']
    return get_reference().get_gene_symbol(gene_id)

def get_variants_for_inheritance_for_project(project, inheritance_mode):
    """
    Get the variants for this project / inheritance combo
    Return dict of family -> list of variants
    """

    # create search specification
    # this could theoretically differ by project, if there are different reference populations
    variant_filter = get_default_variant_filter('moderate_impact')
    variant_filter.ref_freqs.append(('1kg_wgs_phase3', g1k_freq_threshold))
    variant_filter.ref_freqs.append(('1kg_wgs_phase3_popmax', g1k_popmax_freq_threshold))
    variant_filter.ref_freqs.append(('exac_v3', exac_freq_threshold))
    variant_filter.ref_freqs.append(('exac_v3_popmax', exac_popmax_threshold))
    variant_filter.ref_freqs.append(('merck-wgs-3793', merck_wgs_3793_threshold))
    quality_filter = {
        'vcf_filter': 'pass',
        'min_gq': GQ_threshold,
        'min_ab': AB_threshold,
    }

    # run MendelianVariantSearch for each family, collect results

    families = project.get_families()

    for i, family in enumerate(families):
        print("Processing %s - family %s  (%d / %d)" % (inheritance_mode, family.family_id, i+1, len(families)))

        if inheritance_mode == "all_variants":
            yield family, list(get_variants(
                get_datastore(project.project_id),
                family.xfamily(),
                variant_filter=variant_filter,
                quality_filter=quality_filter,
                indivs_to_consider=family.indiv_id_list()
            ))
        else:
            yield family, list(get_variants_with_inheritance_mode(
                get_mall(project.project_id),
                family.xfamily(),
                inheritance_mode,
                variant_filter=variant_filter,
                quality_filter=quality_filter,
            ))

class Command(BaseCommand):

    def handle(self, *args, **options):
        if not args:
            sys.exit("ERROR: please specify project id on the command line")
        if len(args) > 1:
            sys.exit("ERROR: too many args: %s. Only one project id should be provided." % " ".join(args) )

        project_id = args[0]

        # create family_variants.tsv
        family_variants_f = gzip.open('family_variants_%s.tsv.gz' % project_id, 'w')
        writer = csv.writer(family_variants_f, dialect='excel', delimiter='\t')

        header_fields = [
            '#inheritance_mode',
            'project_id',
            'family_id',
            'gene',
            'chrom',
            'pos',
            'ref',
            'alt',
            'rsid',
            'clinvar_status',
            'annotation',
            '1kg_af',
            '1kg_popmax_af',
            'exac_af',
            'exac_popmax_af',
            'merck_wgs_3793_af',
            '',
            ]

        genotype_headers = [
            'sample_id',
            'str',
            'num_alt',
            'allele_balance',
            'AD',
            'DP',
            'GQ',
            'PL',
        ]

        for i in range(0, 10):
            for h in genotype_headers:
                header_fields.append("genotype%d_%s" % (i, h))

        writer.writerow(header_fields)

        for inheritance_mode in ['homozygous_recessive', 'dominant', 'compound_het', 'de_novo', 'x_linked_recessive', 'all_variants']:
            # collect the resources that we'll need here
            annotator = mall.get_annotator()
            custom_population_store = mall.get_custom_population_store()

            project = Project.objects.get(project_id=project_id)

            # get the variants for this inheritance / project combination
            for i, (family, family_results) in enumerate(get_variants_for_inheritance_for_project(project, inheritance_mode)):
                for variant in family_results:
                    custom_populations = custom_population_store.get_frequencies(variant.xpos, variant.ref, variant.alt)
                    g1k_freq = variant.annotation['freqs']['1kg_wgs_phase3']
                    g1k_popmax_freq = variant.annotation['freqs']['1kg_wgs_phase3_popmax']
                    exac_freq = variant.annotation['freqs']['exac_v3']
                    exac_popmax_freq =  variant.annotation['freqs']['exac_v3_popmax']
                    merck_wgs_3793_freq = custom_populations.get('merck-wgs-3793', 0.0)

                    assert g1k_freq <= g1k_freq_threshold, "g1k freq %s > %s" % (g1k_freq, g1k_freq_threshold)
                    assert g1k_popmax_freq <= g1k_popmax_freq_threshold, "g1k freq %s > %s" % (g1k_popmax_freq, g1k_popmax_freq_threshold)
                    assert exac_freq <= exac_freq_threshold, "Exac freq %s > %s" % (exac_freq, exac_freq_threshold)
                    assert exac_popmax_freq <= exac_popmax_threshold, "Exac popmax freq %s > %s" % (exac_popmax_freq, exac_popmax_threshold)
                    assert merck_wgs_3793_freq <= merck_wgs_3793_threshold

                    clinvar_significance = CLINVAR_VARIANTS.get(variant.unique_tuple(), [""])[-1]
                    row = [
                        inheritance_mode,
                        project_id,
                        family.family_id,
                        get_gene_symbol(variant),
                        variant.chr,
                        str(variant.pos),
                        variant.ref,
                        variant.alt,
                        variant.vcf_id,
                        clinvar_significance,
                        variant.annotation['vep_group'],

                        g1k_freq,
                        g1k_popmax_freq,

                        exac_freq,
                        exac_popmax_freq,
                        merck_wgs_3793_freq,
                        '',
                    ]

                    for i, individual in enumerate(family.get_individuals()):
                        if i >= 10:
                            break

                        genotype = variant.get_genotype(individual.indiv_id)

                        if genotype is None:
                            row.extend([individual.indiv_id, "./.", "", "", "", "", "", ""])
                            continue
                        else:
                            assert genotype.filter == "pass", "%s %s - filter is %s " % (variant.chr, variant.pos, genotype.filter)
                            assert genotype.gq >= GQ_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.gq)
                            assert genotype.extras["dp"] >= DP_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.extras["dp"])
                            if genotype.num_alt == 1:
                                assert genotype.ab >= AB_threshold/100., "%s %s - AB is %s " % (variant.chr, variant.pos, genotype.ab)

                            genotype_str = "/".join(genotype.alleles) if genotype.alleles else "./."

                            row.extend([
                                    individual.indiv_id,
                                    genotype_str,
                                    genotype.num_alt,
                                    genotype.ab,
                                    genotype.extras["ad"],
                                    genotype.extras["dp"],
                                    genotype.gq,
                                    genotype.extras["pl"],])

                    writer.writerow(row)
                    family_variants_f.flush()

        family_variants_f.close()
        print("Done")
<commit_msg>Added assert error message<commit_after>import csv
import gzip
import sys
from django.core.management.base import BaseCommand
from xbrowse.core.variant_filters import get_default_variant_filter
from xbrowse.variant_search.family import get_variants_with_inheritance_mode, get_variants
from xbrowse_server import mall
from xbrowse_server.base.models import Project, Family
from xbrowse_server.mall import get_mall, get_reference, get_datastore
from settings import CLINVAR_VARIANTS

AB_threshold = 15
GQ_threshold = 20
DP_threshold = 10
g1k_freq_threshold = 0.01
g1k_popmax_freq_threshold = 0.01
exac_freq_threshold = 0.01
exac_popmax_threshold = 0.01
merck_wgs_3793_threshold = 0.05

def get_gene_symbol(variant):
    gene_id = variant.annotation['vep_annotation'][
        variant.annotation['worst_vep_annotation_index']]['gene']
    return get_reference().get_gene_symbol(gene_id)

def get_variants_for_inheritance_for_project(project, inheritance_mode):
    """
    Get the variants for this project / inheritance combo
    Return dict of family -> list of variants
    """

    # create search specification
    # this could theoretically differ by project, if there are different reference populations
    variant_filter = get_default_variant_filter('moderate_impact')
    variant_filter.ref_freqs.append(('1kg_wgs_phase3', g1k_freq_threshold))
    variant_filter.ref_freqs.append(('1kg_wgs_phase3_popmax', g1k_popmax_freq_threshold))
    variant_filter.ref_freqs.append(('exac_v3', exac_freq_threshold))
    variant_filter.ref_freqs.append(('exac_v3_popmax', exac_popmax_threshold))
    variant_filter.ref_freqs.append(('merck-wgs-3793', merck_wgs_3793_threshold))
    quality_filter = {
        'vcf_filter': 'pass',
        'min_gq': GQ_threshold,
        'min_ab': AB_threshold,
    }

    # run MendelianVariantSearch for each family, collect results

    families = project.get_families()

    for i, family in enumerate(families):
        print("Processing %s - family %s  (%d / %d)" % (inheritance_mode, family.family_id, i+1, len(families)))

        if inheritance_mode == "all_variants":
            yield family, list(get_variants(
                get_datastore(project.project_id),
                family.xfamily(),
                variant_filter=variant_filter,
                quality_filter=quality_filter,
                indivs_to_consider=family.indiv_id_list()
            ))
        else:
            yield family, list(get_variants_with_inheritance_mode(
                get_mall(project.project_id),
                family.xfamily(),
                inheritance_mode,
                variant_filter=variant_filter,
                quality_filter=quality_filter,
            ))

class Command(BaseCommand):

    def handle(self, *args, **options):
        if not args:
            sys.exit("ERROR: please specify project id on the command line")
        if len(args) > 1:
            sys.exit("ERROR: too many args: %s. Only one project id should be provided." % " ".join(args) )

        project_id = args[0]

        # create family_variants.tsv
        family_variants_f = gzip.open('family_variants_%s.tsv.gz' % project_id, 'w')
        writer = csv.writer(family_variants_f, dialect='excel', delimiter='\t')

        header_fields = [
            '#inheritance_mode',
            'project_id',
            'family_id',
            'gene',
            'chrom',
            'pos',
            'ref',
            'alt',
            'rsid',
            'clinvar_status',
            'annotation',
            '1kg_af',
            '1kg_popmax_af',
            'exac_af',
            'exac_popmax_af',
            'merck_wgs_3793_af',
            '',
            ]

        genotype_headers = [
            'sample_id',
            'str',
            'num_alt',
            'allele_balance',
            'AD',
            'DP',
            'GQ',
            'PL',
        ]

        for i in range(0, 10):
            for h in genotype_headers:
                header_fields.append("genotype%d_%s" % (i, h))

        writer.writerow(header_fields)

        for inheritance_mode in ['homozygous_recessive', 'dominant', 'compound_het', 'de_novo', 'x_linked_recessive', 'all_variants']:
            # collect the resources that we'll need here
            annotator = mall.get_annotator()
            custom_population_store = mall.get_custom_population_store()

            project = Project.objects.get(project_id=project_id)

            # get the variants for this inheritance / project combination
            for i, (family, family_results) in enumerate(get_variants_for_inheritance_for_project(project, inheritance_mode)):
                for variant in family_results:
                    custom_populations = custom_population_store.get_frequencies(variant.xpos, variant.ref, variant.alt)
                    g1k_freq = variant.annotation['freqs']['1kg_wgs_phase3']
                    g1k_popmax_freq = variant.annotation['freqs']['1kg_wgs_phase3_popmax']
                    exac_freq = variant.annotation['freqs']['exac_v3']
                    exac_popmax_freq =  variant.annotation['freqs']['exac_v3_popmax']
                    merck_wgs_3793_freq = custom_populations.get('merck-wgs-3793', 0.0)

                    assert g1k_freq <= g1k_freq_threshold, "g1k freq %s > %s" % (g1k_freq, g1k_freq_threshold)
                    assert g1k_popmax_freq <= g1k_popmax_freq_threshold, "g1k freq %s > %s" % (g1k_popmax_freq, g1k_popmax_freq_threshold)
                    assert exac_freq <= exac_freq_threshold, "Exac freq %s > %s" % (exac_freq, exac_freq_threshold)
                    assert exac_popmax_freq <= exac_popmax_threshold, "Exac popmax freq %s > %s" % (exac_popmax_freq, exac_popmax_threshold)
                    assert merck_wgs_3793_freq <= merck_wgs_3793_threshold, "Merck WGS 3793 threshold %s > %s" % (merck_wgs_3793_freq, merck_wgs_3793_threshold)

                    clinvar_significance = CLINVAR_VARIANTS.get(variant.unique_tuple(), [""])[-1]
                    row = [
                        inheritance_mode,
                        project_id,
                        family.family_id,
                        get_gene_symbol(variant),
                        variant.chr,
                        str(variant.pos),
                        variant.ref,
                        variant.alt,
                        variant.vcf_id,
                        clinvar_significance,
                        variant.annotation['vep_group'],

                        g1k_freq,
                        g1k_popmax_freq,

                        exac_freq,
                        exac_popmax_freq,
                        merck_wgs_3793_freq,
                        '',
                    ]

                    for i, individual in enumerate(family.get_individuals()):
                        if i >= 10:
                            break

                        genotype = variant.get_genotype(individual.indiv_id)

                        if genotype is None:
                            row.extend([individual.indiv_id, "./.", "", "", "", "", "", ""])
                            continue
                        else:
                            assert genotype.filter == "pass", "%s %s - filter is %s " % (variant.chr, variant.pos, genotype.filter)
                            assert genotype.gq >= GQ_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.gq)
                            assert genotype.extras["dp"] >= DP_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.extras["dp"])
                            if genotype.num_alt == 1:
                                assert genotype.ab >= AB_threshold/100., "%s %s - AB is %s " % (variant.chr, variant.pos, genotype.ab)

                            genotype_str = "/".join(genotype.alleles) if genotype.alleles else "./."

                            row.extend([
                                    individual.indiv_id,
                                    genotype_str,
                                    genotype.num_alt,
                                    genotype.ab,
                                    genotype.extras["ad"],
                                    genotype.extras["dp"],
                                    genotype.gq,
                                    genotype.extras["pl"],])

                    writer.writerow(row)
                    family_variants_f.flush()

        family_variants_f.close()
        print("Done")
<|endoftext|>


<commit_before># Author: Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD Style.
"""Utilities to build feature vectors from text documents"""

import re
import unicodedata
import numpy as np
import scipy.sparse as sp

ENGLISH_STOP_WORDS = set([
    "a", "about", "above", "across", "after", "afterwards", "again", "against",
    "all", "almost", "alone", "along", "already", "also", "although", "always",
    "am", "among", "amongst", "amoungst", "amount", "an", "and", "another",
    "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
    "around", "as", "at", "back", "be", "became", "because", "become",
    "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
    "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom",
    "but", "by", "call", "can", "cannot", "cant", "co", "computer", "con",
    "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down",
    "due", "during", "each", "eg", "eight", "either", "eleven", "else",
    "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone",
    "everything", "everywhere", "except", "few", "fifteen", "fify", "fill",
    "find", "fire", "first", "five", "for", "former", "formerly", "forty",
    "found", "four", "from", "front", "full", "further", "get", "give", "go",
    "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
    "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
    "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
    "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter",
    "latterly", "least", "less", "ltd", "made", "many", "may", "me",
    "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
    "move", "much", "must", "my", "myself", "name", "namely", "neither", "never",
    "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor",
    "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once",
    "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours",
    "ourselves", "out", "over", "own", "part", "per", "perhaps", "please",
    "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems",
    "serious", "several", "she", "should", "show", "side", "since", "sincere",
    "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime",
    "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than",
    "that", "the", "their", "them", "themselves", "then", "thence", "there",
    "thereafter", "thereby", "therefore", "therein", "thereupon", "these",
    "they", "thick", "thin", "third", "this", "those", "though", "three",
    "through", "throughout", "thru", "thus", "to", "together", "too", "top",
    "toward", "towards", "twelve", "twenty", "two", "un", "under", "until",
    "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what",
    "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas",
    "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while",
    "whither", "who", "whoever", "whole", "whom", "whose", "why", "will",
    "with", "within", "without", "would", "yet", "you", "your", "yours",
    "yourself", "yourselves"])

def strip_accents(s):
    """Transform accentuated unicode symbols into their simple counterpart"""
    return ''.join((c for c in unicodedata.normalize('NFD', s)
                    if unicodedata.category(c) != 'Mn'))

class SimpleAnalyzer(object):
    """Simple analyzer: transform a text document into a sequence of tokens

    This simple implementation does:
        - lower case conversion
        - unicode accents removal
        - token extraction using unicode regexp word bounderies for token of
          minimum size of 2 symbols
    """

    token_pattern = re.compile(r"\b\w\w+\b", re.U)

    def __init__(self, default_charset='utf-8', stop_words=None):
        self.charset = default_charset
        self.stop_words = stop_words

    def analyze(self, text_document):
        if isinstance(text_document, str):
            text_document = text_document.decode(self.charset, 'ignore')
        text_document = strip_accents(text_document.lower())
        tokens = self.token_pattern.findall(text_document)
        if self.stop_words is not None:
            return [w for w in tokens if w not in self.stop_words]
        else:
            return tokens

class HashingVectorizer(object):
    """Compute term frequencies vectors using hashed term space

    See the Hashing-trick related papers referenced by John Langford on this
    page to get a grasp on the usefulness of this representation:

      http://hunch.net/~jl/projects/hash_reps/index.html

    dim is the number of buckets, higher dim means lower collision rate but
    also higher memory requirements and higher processing times on the
    resulting tfidf vectors.

    Documents is a sequence of lists of tokens to initialize the DF estimates.

    TODO handle bigrams in a smart way such as demonstrated here:

      http://streamhacker.com/2010/05/24/text-classification-sentiment-analysis-stopwords-collocations/

    """
    # TODO: implement me using the murmurhash that might be faster: but profile
    # me first :)

    # TODO: make it possible to select between the current dense representation
    # and sparse alternatives from scipy.sparse once the liblinear and libsvm
    # wrappers have been updated to be able to handle it efficiently

    def __init__(self, dim=5000, probes=1, analyzer=SimpleAnalyzer(),
                 use_idf=True):
        self.dim = dim
        self.probes = probes
        self.analyzer = analyzer
        self.use_idf = use_idf

        # start counts at one to avoid zero division while
        # computing IDF
        self.df_counts = np.ones(dim, dtype=long)
        self.tf_vectors = None
        self.sampled = 0

    def hash_sign(self, token, probe=0):
        h = hash(token + (probe * u"#"))
        return abs(h) % self.dim, 1.0 if h % 2 == 0 else -1.0

    def sample_document(self, text, tf_vectors=None, idx=0,
                        update_estimates=True):
        """Extract features from text and update running freq estimates"""
        if tf_vectors is None:
            # allocate term frequency vector and stack to history
            tf_vectors = sp.lil_matrix((1, self.dim))
            stack = True
        else:
            stack = False

        tokens = self.analyzer.analyze(text)
        for token in tokens:
            # TODO add support for cooccurence tokens in a sentence
            # window
            for probe in xrange(self.probes):
                i, incr = self.hash_sign(token, probe)
                tf_vectors[idx, i] += incr
        tf_vectors[idx, :] /= len(tokens) * self.probes

        if update_estimates and self.use_idf:
            # update the running DF estimate
            self.df_counts += np.sum(tf_vectors.nonzero()[0] == idx)
            self.sampled += 1

        if stack:
            if self.tf_vectors is None:
                self.tf_vectors = tf_vectors
            else:
                self.tf_vectors = sp.vstack((self.tf_vectors, tf_vectors))

    def get_idf(self):
        return np.log(float(self.sampled) / self.df_counts)

    def get_tfidf(self):
        """Compute the TF-log(IDF) vectors of the sampled documents"""
        if self.tf_vectors is None:
            return None
        return self.tf_vectors.multiply(self.get_idf()[np.newaxis,:])

    def vectorize(self, document_filepaths):
        """Vectorize a batch of documents"""
        tf_vectors = sp.lil_matrix((len(document_filepaths), self.dim))
        for i, filepath in enumerate(document_filepaths):
            self.sample_document(file(filepath).read(), tf_vectors[i])

        if self.tf_vectors is None:
            self.tf_vectors = tf_vectors
        else:
            self.tf_vectors = sp.vstack((self.tf_vectors, tf_vectors))

    def get_vectors(self):
        if self.use_idf:
            return self.get_tfidf().tocsr()
        else:
            return self.tf_vectors.tocsr()

<commit_msg>using a separate class for the sparse version of the hashing vectorizer<commit_after># Author: Olivier Grisel <olivier.grisel@ensta.org>
#
# License: BSD Style.
"""Utilities to build feature vectors from text documents"""

import re
import unicodedata
import numpy as np
import scipy.sparse as sp

ENGLISH_STOP_WORDS = set([
    "a", "about", "above", "across", "after", "afterwards", "again", "against",
    "all", "almost", "alone", "along", "already", "also", "although", "always",
    "am", "among", "amongst", "amoungst", "amount", "an", "and", "another",
    "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
    "around", "as", "at", "back", "be", "became", "because", "become",
    "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
    "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom",
    "but", "by", "call", "can", "cannot", "cant", "co", "computer", "con",
    "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down",
    "due", "during", "each", "eg", "eight", "either", "eleven", "else",
    "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone",
    "everything", "everywhere", "except", "few", "fifteen", "fify", "fill",
    "find", "fire", "first", "five", "for", "former", "formerly", "forty",
    "found", "four", "from", "front", "full", "further", "get", "give", "go",
    "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
    "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
    "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
    "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter",
    "latterly", "least", "less", "ltd", "made", "many", "may", "me",
    "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
    "move", "much", "must", "my", "myself", "name", "namely", "neither", "never",
    "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor",
    "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once",
    "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours",
    "ourselves", "out", "over", "own", "part", "per", "perhaps", "please",
    "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems",
    "serious", "several", "she", "should", "show", "side", "since", "sincere",
    "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime",
    "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than",
    "that", "the", "their", "them", "themselves", "then", "thence", "there",
    "thereafter", "thereby", "therefore", "therein", "thereupon", "these",
    "they", "thick", "thin", "third", "this", "those", "though", "three",
    "through", "throughout", "thru", "thus", "to", "together", "too", "top",
    "toward", "towards", "twelve", "twenty", "two", "un", "under", "until",
    "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what",
    "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas",
    "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while",
    "whither", "who", "whoever", "whole", "whom", "whose", "why", "will",
    "with", "within", "without", "would", "yet", "you", "your", "yours",
    "yourself", "yourselves"])

def strip_accents(s):
    """Transform accentuated unicode symbols into their simple counterpart"""
    return ''.join((c for c in unicodedata.normalize('NFD', s)
                    if unicodedata.category(c) != 'Mn'))

class SimpleAnalyzer(object):
    """Simple analyzer: transform a text document into a sequence of tokens

    This simple implementation does:
        - lower case conversion
        - unicode accents removal
        - token extraction using unicode regexp word bounderies for token of
          minimum size of 2 symbols
    """

    token_pattern = re.compile(r"\b\w\w+\b", re.U)

    def __init__(self, default_charset='utf-8', stop_words=None):
        self.charset = default_charset
        self.stop_words = stop_words

    def analyze(self, text_document):
        if isinstance(text_document, str):
            text_document = text_document.decode(self.charset, 'ignore')
        text_document = strip_accents(text_document.lower())
        tokens = self.token_pattern.findall(text_document)
        if self.stop_words is not None:
            return [w for w in tokens if w not in self.stop_words]
        else:
            return tokens

class SparseHashingVectorizer(object):
    """Compute term frequencies vectors using hashed term space in sparse matrix

    The logic is the same as HashingVectorizer but it is possible to use much
    larger dimension vectors without memory issues thanks to the usage of
    scipy.sparse datastructure to store the tf vectors.
    """

    def __init__(self, dim=50000, probes=1, analyzer=SimpleAnalyzer(),
                 use_idf=True):
        self.dim = dim
        self.probes = probes
        self.analyzer = analyzer
        self.use_idf = use_idf

        # start counts at one to avoid zero division while
        # computing IDF
        self.df_counts = np.ones(dim, dtype=long)
        self.tf_vectors = None
        self.sampled = 0

    def hash_sign(self, token, probe=0):
        h = hash(token + (probe * u"#"))
        return abs(h) % self.dim, 1.0 if h % 2 == 0 else -1.0

    def sample_document(self, text, tf_vectors=None, idx=0,
                        update_estimates=True):
        """Extract features from text and update running freq estimates"""
        if tf_vectors is None:
            # allocate term frequency vector and stack to history
            tf_vectors = sp.lil_matrix((1, self.dim))
            stack = True
        else:
            stack = False

        tokens = self.analyzer.analyze(text)
        for token in tokens:
            # TODO add support for cooccurence tokens in a sentence
            # window
            for probe in xrange(self.probes):
                i, incr = self.hash_sign(token, probe)
                tf_vectors[idx, i] += incr
        tf_vectors[idx, :] /= len(tokens) * self.probes

        if update_estimates and self.use_idf:
            # update the running DF estimate
            self.df_counts += np.sum(tf_vectors.nonzero()[0] == idx)
            self.sampled += 1

        if stack:
            if self.tf_vectors is None:
                self.tf_vectors = tf_vectors
            else:
                self.tf_vectors = sp.vstack((self.tf_vectors, tf_vectors))

    def get_idf(self):
        return np.log(float(self.sampled) / self.df_counts)

    def get_tfidf(self):
        """Compute the TF-log(IDF) vectors of the sampled documents"""
        if self.tf_vectors is None:
            return None
        return self.tf_vectors.multiply(self.get_idf()[np.newaxis,:])

    def vectorize(self, document_filepaths):
        """Vectorize a batch of documents"""
        tf_vectors = sp.lil_matrix((len(document_filepaths), self.dim))
        for i, filepath in enumerate(document_filepaths):
            self.sample_document(file(filepath).read(), tf_vectors[i])

        if self.tf_vectors is None:
            self.tf_vectors = tf_vectors
        else:
            self.tf_vectors = sp.vstack((self.tf_vectors, tf_vectors))

    def get_vectors(self):
        if self.use_idf:
            return self.get_tfidf().tocsr()
        else:
            return self.tf_vectors.tocsr()

<|endoftext|>


<commit_before>import morepath
from .app import App

def run():
    morepath.autosetup()
    morepath.run(App())
<commit_msg>Added main clause to package_name.__main__<commit_after>import morepath
from .app import App

def run():
    morepath.autosetup()
    morepath.run(App())

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>"""Trust-region optimization."""
from __future__ import division, print_function, absolute_import

import math

import numpy as np
import scipy.linalg
from .optimize import (_check_unknown_options, wrap_function, _status_message,
                       OptimizeResult)

__all__ = []

class BaseQuadraticSubproblem(object):
    """
    Base/abstract class defining the quadratic model for trust-region
    minimization. Child classes must implement the ``solve`` method.

    Values of the objective function, jacobian and hessian (if provided) at
    the current iterate ``x`` are evaluated on demand and then stored as
    attributes ``fun``, ``jac``, ``hess``.
    """

    def __init__(self, x, fun, jac, hess=None, hessp=None):
        self._x = x
        self._f = None
        self._g = None
        self._h = None
        self._g_mag = None
        self._cauchy_point = None
        self._newton_point = None
        self._fun = fun
        self._jac = jac
        self._hess = hess
        self._hessp = hessp

    def __call__(self, p):
        return self.fun + np.dot(self.jac, p) + 0.5 * np.dot(p, self.hessp(p))

    @property
    def fun(self):
        """Value of objective function at current iteration."""
        if self._f is None:
            self._f = self._fun(self._x)
        return self._f

    @property
    def jac(self):
        """Value of jacobian of objective function at current iteration."""
        if self._g is None:
            self._g = self._jac(self._x)
        return self._g

    @property
    def hess(self):
        """Value of hessian of objective function at current iteration."""
        if self._h is None:
            self._h = self._hess(self._x)
        return self._h

    def hessp(self, p):
        if self._hessp is not None:
            return self._hessp(self._x, p)
        else:
            return np.dot(self.hess, p)

    @property
    def jac_mag(self):
        """Magniture of jacobian of objective function at current iteration."""
        if self._g_mag is None:
            self._g_mag = scipy.linalg.norm(self.jac)
        return self._g_mag

    def get_boundaries_intersections(self, z, d, trust_radius):
        """
        Solve the scalar quadratic equation ||z + t d|| == trust_radius.
        This is like a line-sphere intersection.
        Return the two values of t, sorted from low to high.
        """
        a = np.dot(d, d)
        b = 2 * np.dot(z, d)
        c = np.dot(z, z) - trust_radius**2
        sqrt_discriminant = math.sqrt(b*b - 4*a*c)

        # The following calculation is mathematically
        # equivalent to:
        # ta = (-b - sqrt_discriminant) / (2*a)
        # tb = (-b + sqrt_discriminant) / (2*a)
        # but produce smaller round off errors.
        # Look at Matrix Computation p.97
        # for a better justification.
        aux = b + math.copysign(sqrt_discriminant, b)
        ta = -aux / (2*a)
        tb = -2*c / aux
        return sorted([ta, tb])

    def solve(self, trust_radius):
        raise NotImplementedError('The solve method should be implemented by '
                                  'the child class')

def _minimize_trust_region(fun, x0, args=(), jac=None, hess=None, hessp=None,
                           subproblem=None, initial_trust_radius=1.0,
                           max_trust_radius=1000.0, eta=0.15, gtol=1e-4,
                           maxiter=None, disp=False, return_all=False,
                           callback=None, inexact=True, **unknown_options):
    """
    Minimization of scalar function of one or more variables using a
    trust-region algorithm.

    Options for the trust-region algorithm are:
        initial_trust_radius : float
            Initial trust radius.
        max_trust_radius : float
            Never propose steps that are longer than this value.
        eta : float
            Trust region related acceptance stringency for proposed steps.
        gtol : float
            Gradient norm must be less than `gtol`
            before successful termination.
        maxiter : int
            Maximum number of iterations to perform.
        disp : bool
            If True, print convergence message.
        inexact : bool
            Accuracy to solve subproblems. If True requires less nonlinear
            iterations, but more vector products. Only effective for method
            trust-krylov.

    This function is called by the `minimize` function.
    It is not supposed to be called directly.
    """
    _check_unknown_options(unknown_options)

    if jac is None:
        raise ValueError('Jacobian is currently required for trust-region '
                         'methods')
    if hess is None and hessp is None:
        raise ValueError('Either the Hessian or the Hessian-vector product '
                         'is currently required for trust-region methods')
    if subproblem is None:
        raise ValueError('A subproblem solving strategy is required for '
                         'trust-region methods')
    if not (0 <= eta < 0.25):
        raise Exception('invalid acceptance stringency')
    if max_trust_radius <= 0:
        raise Exception('the max trust radius must be positive')
    if initial_trust_radius <= 0:
        raise ValueError('the initial trust radius must be positive')
    if initial_trust_radius >= max_trust_radius:
        raise ValueError('the initial trust radius must be less than the '
                         'max trust radius')

    # force the initial guess into a nice format
    x0 = np.asarray(x0).flatten()

    # Wrap the functions, for a couple reasons.
    # This tracks how many times they have been called
    # and it automatically passes the args.
    nfun, fun = wrap_function(fun, args)
    njac, jac = wrap_function(jac, args)
    nhess, hess = wrap_function(hess, args)
    nhessp, hessp = wrap_function(hessp, args)

    # limit the number of iterations
    if maxiter is None:
        maxiter = len(x0)*200

    # init the search status
    warnflag = 0

    # initialize the search
    trust_radius = initial_trust_radius
    x = x0
    if return_all:
        allvecs = [x]
    m = subproblem(x, fun, jac, hess, hessp)
    k = 0

    # search for the function min
    # do not even start if the gradient is small enough
    while m.jac_mag >= gtol:

        # Solve the sub-problem.
        # This gives us the proposed step relative to the current position
        # and it tells us whether the proposed step
        # has reached the trust region boundary or not.
        try:
            p, hits_boundary = m.solve(trust_radius)
        except np.linalg.linalg.LinAlgError as e:
            warnflag = 3
            break

        # calculate the predicted value at the proposed point
        predicted_value = m(p)

        # define the local approximation at the proposed point
        x_proposed = x + p
        m_proposed = subproblem(x_proposed, fun, jac, hess, hessp)

        # evaluate the ratio defined in equation (4.4)
        actual_reduction = m.fun - m_proposed.fun
        predicted_reduction = m.fun - predicted_value
        if predicted_reduction <= 0:
            warnflag = 2
            break
        rho = actual_reduction / predicted_reduction

        # update the trust radius according to the actual/predicted ratio
        if rho < 0.25:
            trust_radius *= 0.25
        elif rho > 0.75 and hits_boundary:
            trust_radius = min(2*trust_radius, max_trust_radius)

        # if the ratio is high enough then accept the proposed step
        if rho > eta:
            x = x_proposed
            m = m_proposed

        # append the best guess, call back, increment the iteration count
        if return_all:
            allvecs.append(np.copy(x))
        if callback is not None:
            callback(np.copy(x))
        k += 1

        # check if the gradient is small enough to stop
        if m.jac_mag < gtol:
            warnflag = 0
            break

        # check if we have looked at enough iterations
        if k >= maxiter:
            warnflag = 1
            break

    # print some stuff if requested
    status_messages = (
            _status_message['success'],
            _status_message['maxiter'],
            'A bad approximation caused failure to predict improvement.',
            'A linalg error occurred, such as a non-psd Hessian.',
            )
    if disp:
        if warnflag == 0:
            print(status_messages[warnflag])
        else:
            print('Warning: ' + status_messages[warnflag])
        print("         Current function value: %f" % m.fun)
        print("         Iterations: %d" % k)
        print("         Function evaluations: %d" % nfun[0])
        print("         Gradient evaluations: %d" % njac[0])
        print("         Hessian evaluations: %d" % (nhess[0] + nhessp[0]))

    result = OptimizeResult(x=x, success=(warnflag == 0), status=warnflag,
                            fun=m.fun, jac=m.jac, nfev=nfun[0], njev=njac[0],
                            nhev=nhess[0] + nhessp[0], nit=k,
                            message=status_messages[warnflag])

    if hess is not None:
        result['hess'] = m.hess

    if return_all:
        result['allvecs'] = allvecs

    return result
<commit_msg>MAINT: trust-* uses ScalarFunction<commit_after>"""Trust-region optimization."""
from __future__ import division, print_function, absolute_import

import math

import numpy as np
import scipy.linalg
from .optimize import (_check_unknown_options, wrap_function, _status_message,
                       OptimizeResult, _prepare_scalar_function)

__all__ = []

class BaseQuadraticSubproblem(object):
    """
    Base/abstract class defining the quadratic model for trust-region
    minimization. Child classes must implement the ``solve`` method.

    Values of the objective function, jacobian and hessian (if provided) at
    the current iterate ``x`` are evaluated on demand and then stored as
    attributes ``fun``, ``jac``, ``hess``.
    """

    def __init__(self, x, fun, jac, hess=None, hessp=None):
        self._x = x
        self._f = None
        self._g = None
        self._h = None
        self._g_mag = None
        self._cauchy_point = None
        self._newton_point = None
        self._fun = fun
        self._jac = jac
        self._hess = hess
        self._hessp = hessp

    def __call__(self, p):
        return self.fun + np.dot(self.jac, p) + 0.5 * np.dot(p, self.hessp(p))

    @property
    def fun(self):
        """Value of objective function at current iteration."""
        if self._f is None:
            self._f = self._fun(self._x)
        return self._f

    @property
    def jac(self):
        """Value of jacobian of objective function at current iteration."""
        if self._g is None:
            self._g = self._jac(self._x)
        return self._g

    @property
    def hess(self):
        """Value of hessian of objective function at current iteration."""
        if self._h is None:
            self._h = self._hess(self._x)
        return self._h

    def hessp(self, p):
        if self._hessp is not None:
            return self._hessp(self._x, p)
        else:
            return np.dot(self.hess, p)

    @property
    def jac_mag(self):
        """Magnitude of jacobian of objective function at current iteration."""
        if self._g_mag is None:
            self._g_mag = scipy.linalg.norm(self.jac)
        return self._g_mag

    def get_boundaries_intersections(self, z, d, trust_radius):
        """
        Solve the scalar quadratic equation ||z + t d|| == trust_radius.
        This is like a line-sphere intersection.
        Return the two values of t, sorted from low to high.
        """
        a = np.dot(d, d)
        b = 2 * np.dot(z, d)
        c = np.dot(z, z) - trust_radius**2
        sqrt_discriminant = math.sqrt(b*b - 4*a*c)

        # The following calculation is mathematically
        # equivalent to:
        # ta = (-b - sqrt_discriminant) / (2*a)
        # tb = (-b + sqrt_discriminant) / (2*a)
        # but produce smaller round off errors.
        # Look at Matrix Computation p.97
        # for a better justification.
        aux = b + math.copysign(sqrt_discriminant, b)
        ta = -aux / (2*a)
        tb = -2*c / aux
        return sorted([ta, tb])

    def solve(self, trust_radius):
        raise NotImplementedError('The solve method should be implemented by '
                                  'the child class')

def _minimize_trust_region(fun, x0, args=(), jac=None, hess=None, hessp=None,
                           subproblem=None, initial_trust_radius=1.0,
                           max_trust_radius=1000.0, eta=0.15, gtol=1e-4,
                           maxiter=None, disp=False, return_all=False,
                           callback=None, inexact=True, **unknown_options):
    """
    Minimization of scalar function of one or more variables using a
    trust-region algorithm.

    Options for the trust-region algorithm are:
        initial_trust_radius : float
            Initial trust radius.
        max_trust_radius : float
            Never propose steps that are longer than this value.
        eta : float
            Trust region related acceptance stringency for proposed steps.
        gtol : float
            Gradient norm must be less than `gtol`
            before successful termination.
        maxiter : int
            Maximum number of iterations to perform.
        disp : bool
            If True, print convergence message.
        inexact : bool
            Accuracy to solve subproblems. If True requires less nonlinear
            iterations, but more vector products. Only effective for method
            trust-krylov.

    This function is called by the `minimize` function.
    It is not supposed to be called directly.
    """
    _check_unknown_options(unknown_options)

    if jac is None:
        raise ValueError('Jacobian is currently required for trust-region '
                         'methods')
    if hess is None and hessp is None:
        raise ValueError('Either the Hessian or the Hessian-vector product '
                         'is currently required for trust-region methods')
    if subproblem is None:
        raise ValueError('A subproblem solving strategy is required for '
                         'trust-region methods')
    if not (0 <= eta < 0.25):
        raise Exception('invalid acceptance stringency')
    if max_trust_radius <= 0:
        raise Exception('the max trust radius must be positive')
    if initial_trust_radius <= 0:
        raise ValueError('the initial trust radius must be positive')
    if initial_trust_radius >= max_trust_radius:
        raise ValueError('the initial trust radius must be less than the '
                         'max trust radius')

    # force the initial guess into a nice format
    x0 = np.asarray(x0).flatten()

    # A ScalarFunction representing the problem. This caches calls to fun, jac,
    # hess.
    sf = _prepare_scalar_function(fun, x0, jac=jac, hess=hess, args=args)
    fun = sf.fun
    jac = sf.grad
    if hess is not None:
        hess = sf.hess
    # ScalarFunction doesn't represent hessp
    nhessp, hessp = wrap_function(hessp, args)

    # limit the number of iterations
    if maxiter is None:
        maxiter = len(x0)*200

    # init the search status
    warnflag = 0

    # initialize the search
    trust_radius = initial_trust_radius
    x = x0
    if return_all:
        allvecs = [x]
    m = subproblem(x, fun, jac, hess, hessp)
    k = 0

    # search for the function min
    # do not even start if the gradient is small enough
    while m.jac_mag >= gtol:

        # Solve the sub-problem.
        # This gives us the proposed step relative to the current position
        # and it tells us whether the proposed step
        # has reached the trust region boundary or not.
        try:
            p, hits_boundary = m.solve(trust_radius)
        except np.linalg.linalg.LinAlgError as e:
            warnflag = 3
            break

        # calculate the predicted value at the proposed point
        predicted_value = m(p)

        # define the local approximation at the proposed point
        x_proposed = x + p
        m_proposed = subproblem(x_proposed, fun, jac, hess, hessp)

        # evaluate the ratio defined in equation (4.4)
        actual_reduction = m.fun - m_proposed.fun
        predicted_reduction = m.fun - predicted_value
        if predicted_reduction <= 0:
            warnflag = 2
            break
        rho = actual_reduction / predicted_reduction

        # update the trust radius according to the actual/predicted ratio
        if rho < 0.25:
            trust_radius *= 0.25
        elif rho > 0.75 and hits_boundary:
            trust_radius = min(2*trust_radius, max_trust_radius)

        # if the ratio is high enough then accept the proposed step
        if rho > eta:
            x = x_proposed
            m = m_proposed

        # append the best guess, call back, increment the iteration count
        if return_all:
            allvecs.append(np.copy(x))
        if callback is not None:
            callback(np.copy(x))
        k += 1

        # check if the gradient is small enough to stop
        if m.jac_mag < gtol:
            warnflag = 0
            break

        # check if we have looked at enough iterations
        if k >= maxiter:
            warnflag = 1
            break

    # print some stuff if requested
    status_messages = (
            _status_message['success'],
            _status_message['maxiter'],
            'A bad approximation caused failure to predict improvement.',
            'A linalg error occurred, such as a non-psd Hessian.',
            )
    if disp:
        if warnflag == 0:
            print(status_messages[warnflag])
        else:
            print('Warning: ' + status_messages[warnflag])
        print("         Current function value: %f" % m.fun)
        print("         Iterations: %d" % k)
        print("         Function evaluations: %d" % sf.nfev)
        print("         Gradient evaluations: %d" % sf.ngev)
        print("         Hessian evaluations: %d" % (sf.nhev + nhessp[0]))

    result = OptimizeResult(x=x, success=(warnflag == 0), status=warnflag,
                            fun=m.fun, jac=m.jac, nfev=sf.nfev, njev=sf.ngev,
                            nhev=sf.nhev + nhessp[0], nit=k,
                            message=status_messages[warnflag])

    if hess is not None:
        result['hess'] = m.hess

    if return_all:
        result['allvecs'] = allvecs

    return result
<|endoftext|>


<commit_before># This script is used to parse BOOST special function test data into something
# we can easily import in numpy.
import re
import os

# Where to put the data (directory will be created)
DATA_DIR = 'scipy/special/tests/data/boost'
# Where to pull out boost data
BOOST_SRC = "boostmath/test"

CXX_COMMENT = re.compile(r'^\s+//')
DATA_REGEX = re.compile(r'^\s*/*\{*\s*SC_')
ITEM_REGEX = re.compile(r'[+-]?\d*\.?\d+(?:[eE][+-]?\d+)?')
HEADER_REGEX = re.compile(
r'const boost::array\<boost::array\<.*, (\d+)\>, (\d+)\> ([a-zA-Z_\d]+)')

IGNORE_PATTERNS = [
    # Makes use of ldexp and casts
    "hypergeometric_1F1_big_double_limited.ipp",
    "hypergeometric_1F1_big_unsolved.ipp",

    # Makes use of numeric_limits and ternary operator
    "beta_small_data.ipp",

    # Doesn't contain any data
    "almost_equal.ipp",

    # Derivatives functions don't exist
    "bessel_y01_prime_data.ipp",
    "bessel_yn_prime_data.ipp",
    "sph_bessel_prime_data.ipp",
    "sph_neumann_prime_data.ipp",

    # Data files not needed by scipy special tests.
    "ibeta_derivative_",
    r"ellint_r[cdfjg]_[^d]",
    r"ellint_d2?_",
    "jacobi_",
    "heuman_lambda_",
    "hypergeometric_",
    "nct_",
    r".*gammap1m1_",
    "trig_",
    "powm1_data.ipp",
]

def _raw_data(line):
    items = line.split(',')
    l = []
    for item in items:
        m = ITEM_REGEX.search(item)
        if m:
            q = m.group(0)
            l.append(q)
    return l

def parse_ipp_file(filename):
    print(filename)
    a = open(filename, 'r')
    lines = a.readlines()
    data = {}
    i = 0
    while (i < len(lines)):
        line = lines[i]
        m = HEADER_REGEX.search(line)
        if m:
            d = int(m.group(1))
            n = int(m.group(2))
            print("d = {0}, n = {1}".format(d, n))
            cdata = []
            i += 1
            line = lines[i]
            # Skip comments
            while CXX_COMMENT.match(line):
                i += 1
                line = lines[i]
            while DATA_REGEX.match(line):
                cdata.append(_raw_data(line))
                i += 1
                line = lines[i]
                # Skip comments
                while CXX_COMMENT.match(line):
                    i += 1
                    line = lines[i]
            if not len(cdata) == n:
                raise ValueError("parsed data: %d, expected %d" % (len(cdata), n))
            data[m.group(3)] = cdata
        else:
            i += 1

    return data

def dump_dataset(filename, data):
    fid = open(filename, 'w')
    try:
        for line in data:
            fid.write("%s\n" % " ".join(line))
    finally:
        fid.close()

def dump_datasets(filename):
    base, ext = os.path.splitext(os.path.basename(filename))
    base += '_%s' % ext[1:]
    datadir = os.path.join(DATA_DIR, base)
    os.makedirs(datadir)
    datasets = parse_ipp_file(filename)
    for k, d in datasets.items():
        print(k, len(d))
        dfilename = os.path.join(datadir, k) + '.txt'
        dump_dataset(dfilename, d)

if __name__ == '__main__':
    for filename in sorted(os.listdir(BOOST_SRC)):
        # Note: Misses data in hpp files (e.x. powm1_sqrtp1m1_test.hpp)
        if filename.endswith(".ipp"):
            if any(re.match(pattern, filename) for pattern in IGNORE_PATTERNS):
                continue

            path = os.path.join(BOOST_SRC, filename)
            print("================= %s ===============" % path)
            dump_datasets(path)
<commit_msg>file reading with-block<commit_after># This script is used to parse BOOST special function test data into something
# we can easily import in numpy.
import re
import os

# Where to put the data (directory will be created)
DATA_DIR = 'scipy/special/tests/data/boost'
# Where to pull out boost data
BOOST_SRC = "boostmath/test"

CXX_COMMENT = re.compile(r'^\s+//')
DATA_REGEX = re.compile(r'^\s*/*\{*\s*SC_')
ITEM_REGEX = re.compile(r'[+-]?\d*\.?\d+(?:[eE][+-]?\d+)?')
HEADER_REGEX = re.compile(
r'const boost::array\<boost::array\<.*, (\d+)\>, (\d+)\> ([a-zA-Z_\d]+)')

IGNORE_PATTERNS = [
    # Makes use of ldexp and casts
    "hypergeometric_1F1_big_double_limited.ipp",
    "hypergeometric_1F1_big_unsolved.ipp",

    # Makes use of numeric_limits and ternary operator
    "beta_small_data.ipp",

    # Doesn't contain any data
    "almost_equal.ipp",

    # Derivatives functions don't exist
    "bessel_y01_prime_data.ipp",
    "bessel_yn_prime_data.ipp",
    "sph_bessel_prime_data.ipp",
    "sph_neumann_prime_data.ipp",

    # Data files not needed by scipy special tests.
    "ibeta_derivative_",
    r"ellint_r[cdfjg]_[^d]",
    r"ellint_d2?_",
    "jacobi_",
    "heuman_lambda_",
    "hypergeometric_",
    "nct_",
    r".*gammap1m1_",
    "trig_",
    "powm1_data.ipp",
]

def _raw_data(line):
    items = line.split(',')
    l = []
    for item in items:
        m = ITEM_REGEX.search(item)
        if m:
            q = m.group(0)
            l.append(q)
    return l

def parse_ipp_file(filename):
    print(filename)
    with open(filename, 'r') as a:
        lines = a.readlines()
    data = {}
    i = 0
    while (i < len(lines)):
        line = lines[i]
        m = HEADER_REGEX.search(line)
        if m:
            d = int(m.group(1))
            n = int(m.group(2))
            print(f"d = {d}, n = {n}")
            cdata = []
            i += 1
            line = lines[i]
            # Skip comments
            while CXX_COMMENT.match(line):
                i += 1
                line = lines[i]
            while DATA_REGEX.match(line):
                cdata.append(_raw_data(line))
                i += 1
                line = lines[i]
                # Skip comments
                while CXX_COMMENT.match(line):
                    i += 1
                    line = lines[i]
            if not len(cdata) == n:
                raise ValueError(f"parsed data: {len(cdata)}, expected {n}")
            data[m.group(3)] = cdata
        else:
            i += 1

    return data

def dump_dataset(filename, data):
    fid = open(filename, 'w')
    try:
        for line in data:
            fid.write("%s\n" % " ".join(line))
    finally:
        fid.close()

def dump_datasets(filename):
    base, ext = os.path.splitext(os.path.basename(filename))
    base += '_%s' % ext[1:]
    datadir = os.path.join(DATA_DIR, base)
    os.makedirs(datadir)
    datasets = parse_ipp_file(filename)
    for k, d in datasets.items():
        print(k, len(d))
        dfilename = os.path.join(datadir, k) + '.txt'
        dump_dataset(dfilename, d)

if __name__ == '__main__':
    for filename in sorted(os.listdir(BOOST_SRC)):
        # Note: Misses data in hpp files (e.x. powm1_sqrtp1m1_test.hpp)
        if filename.endswith(".ipp"):
            if any(re.match(pattern, filename) for pattern in IGNORE_PATTERNS):
                continue

            path = os.path.join(BOOST_SRC, filename)
            print(f"================= {path} ===============")
            dump_datasets(path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, sys
from glob import glob
from scipy_distutils.core import Extension
from scipy_distutils.misc_util import get_path, default_config_dict,dot_join
import shutil

def configuration(parent_package=''):
    parent_path = parent_package
    if parent_package:
        parent_package += '.'
    local_path = get_path(__name__)

    config = default_config_dict()
    config['packages'].append(parent_package+'scipy_base')
    config['package_dir'][parent_package+'scipy_base'] = local_path

    config['packages'].append(dot_join(parent_package,'scipy_base.tests'))
    test_path = os.path.join(local_path,'tests')
    config['package_dir']['scipy_base.tests'] = test_path

    # scipy_base.fastumath module
    sources = ['scipy_base.fastumathmodule.c','isnan.c']
    sources = [os.path.join(local_path,x) for x in sources]
    ext = Extension('scipy_base.fastumath',sources,libraries=[])
    config['ext_modules'].append(ext)

    # Test to see if big or little-endian machine and get correct default
    #   mconf.h module.
    if sys.byteorder == "little":
        print "### Little Endian detected ####"
        shutil.copy2(os.path.join(local_path,'mconf_lite_LE.h'),os.path.join(local_path,'mconf_lite.h'))
    else:
        print "### Big Endian detected ####"
        shutil.copy2(os.path.join(local_path,'mconf_lite_BE.h'),os.path.join(local_path,'mconf_lite.h'))

    return config

if __name__ == '__main__':
    from scipy_distutils.core import setup
    setup(**configuration())
<commit_msg>Fixed scipy_base.fastumath<commit_after>#!/usr/bin/env python

import os, sys
from glob import glob
from scipy_distutils.core import Extension
from scipy_distutils.misc_util import get_path, default_config_dict,dot_join
import shutil

def configuration(parent_package=''):
    parent_path = parent_package
    if parent_package:
        parent_package += '.'
    local_path = get_path(__name__)

    config = default_config_dict()
    config['packages'].append(parent_package+'scipy_base')
    config['package_dir'][parent_package+'scipy_base'] = local_path

    config['packages'].append(dot_join(parent_package,'scipy_base.tests'))
    test_path = os.path.join(local_path,'tests')
    config['package_dir']['scipy_base.tests'] = test_path

    # scipy_base.fastumath module
    sources = ['fastumathmodule.c','isnan.c']
    sources = [os.path.join(local_path,x) for x in sources]
    ext = Extension('scipy_base.fastumath',sources,libraries=[])
    config['ext_modules'].append(ext)

    # Test to see if big or little-endian machine and get correct default
    #   mconf.h module.
    if sys.byteorder == "little":
        print "### Little Endian detected ####"
        shutil.copy2(os.path.join(local_path,'mconf_lite_LE.h'),os.path.join(local_path,'mconf_lite.h'))
    else:
        print "### Big Endian detected ####"
        shutil.copy2(os.path.join(local_path,'mconf_lite_BE.h'),os.path.join(local_path,'mconf_lite.h'))

    return config

if __name__ == '__main__':
    from scipy_distutils.core import setup
    setup(**configuration())
<|endoftext|>


<commit_before># This file is automatically updated with get_version
# function from scipy_distutils.misc_utils.py
version = '0.6.23-alpha-72'
version_info = (0, 6, 23, 'alpha', 72)
<commit_msg>version update<commit_after># This file is automatically updated with update_version
# function from scipy_distutils.misc_utils.py
version = '0.6.23-alpha-81'
version_info = (0, 6, 23, 'alpha', 81)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from odoo.tests.common import TransactionCase
from odoo.exceptions import UserError, ValidationError

class test_process(TransactionCase):

    def test_create(self):
        """新建审批配置规则，如果配置的模型有type字段而规则未输入type，保存时给出提示"""
        with self.assertRaises(ValidationError):
            self.env['good_process.process'].create({
                'model_id': self.env.ref('buy.model_buy_order').id,
            })

class test_mail_thread(TransactionCase):

    def setUp(self):
        '''准备基本数据'''
        super(test_mail_thread, self).setUp()
        self.approve_rule = self.browse_ref('scm.process_buy_order') # 审批规则
        self.order = self.env.ref('buy.buy_order_1').copy()
        self.staff_admin = self.env.ref('staff.staff_1')
        self.user_demo = self.browse_ref('base.user_demo')

    def test_normal_case(self):
        """正常审批流程"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)

        self.assertTrue(self.order._approve_state == u'已提交')
        # 经理审批
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'审批中')
        # 自己先拒绝
        self.order.good_process_refused(self.order.id, self.order._name)
        # 经理重新审批
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        # 自己审批
        self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'已审批')
        self.order.buy_order_done()
        self.order.buy_order_draft()
        self.order.unlink()

    def test_approver_sequence(self):
        """审批顺序"""
        group_1 = self.env.ref('scm.group_process_buy_order')
        # 自己审批
        result = self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(result[0] == u'您不是这张单据的下一个审批者')

        # admin的经理改为空，将用户组1中的用户改为Alice,admin去审批
        self.staff_admin.parent_id = False  #TODO:不起作用
        group_1.write({'users': [(6, 0, [self.env.ref('scm.user_alice').id])]})
        res = self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(res[0] == u'您不是这张单据的下一个审批者')

    def test_refuser_sequence(self):
        """拒绝顺序"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)
        # 自己拒绝
        result = self.order.good_process_refused(self.order.id, self.order._name)
        self.assertTrue(result[0] == u'您是第一批需要审批的人，无需拒绝！')

        # 经理和自己审批之后
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.order.good_process_approve(self.order.id, self.order._name)
        res = self.order.good_process_refused(self.order.id, self.order._name)
        self.assertTrue(res[0] == u'已经通过不能拒绝！')

    def test_unlink(self):
        """级联删除"""
        self.order.unlink()

    def test_write(self):
        """write 审批异常流程"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)

        # 已提交时审核报错
        self.assertTrue(self.order._approve_state == u'已提交')
        with self.assertRaises(ValidationError):
            self.order.buy_order_done()
        # 经理审批，审批中审核报错
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'审批中')
        with self.assertRaises(ValidationError):
            self.order.buy_order_done()
        # 审批中修改其他字段报错，不可删除
        with self.assertRaises(ValidationError):
            self.order.date = '2017-04-19'
        with self.assertRaises(ValidationError):
            self.order.unlink()
        # 已审批不可修改,不可删除
        self.order.good_process_approve(self.order.id, self.order._name)
        with self.assertRaises(ValidationError):
            self.order.date = '2017-04-19'
        with self.assertRaises(ValidationError):
            self.order.unlink()
        # 已审核的单据不可删除
        self.order.buy_order_done()
        with self.assertRaises(UserError):
            self.order.unlink()

class test_approver(TransactionCase):

    def setUp(self):
        '''准备基本数据'''
        super(test_approver, self).setUp()
        self.approve_rule = self.browse_ref('scm.process_buy_order') # 审批规则
        self.order = self.env.ref('buy.buy_order_1').copy()
        self.user_demo = self.browse_ref('base.user_demo')

    def test_goto(self):
        """查看并处理"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)
        self.order.with_env(env2)._to_approver_ids[0].goto()
<commit_msg>[FIX]去掉无法执行的删除操作以修复ci<commit_after># -*- coding: utf-8 -*-
from odoo.tests.common import TransactionCase
from odoo.exceptions import UserError, ValidationError

class test_process(TransactionCase):

    def test_create(self):
        """新建审批配置规则，如果配置的模型有type字段而规则未输入type，保存时给出提示"""
        with self.assertRaises(ValidationError):
            self.env['good_process.process'].create({
                'model_id': self.env.ref('buy.model_buy_order').id,
            })

class test_mail_thread(TransactionCase):

    def setUp(self):
        '''准备基本数据'''
        super(test_mail_thread, self).setUp()
        self.approve_rule = self.browse_ref('scm.process_buy_order') # 审批规则
        self.order = self.env.ref('buy.buy_order_1').copy()
        self.staff_admin = self.env.ref('staff.staff_1')
        self.user_demo = self.browse_ref('base.user_demo')

    def test_normal_case(self):
        """正常审批流程"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)

        self.assertTrue(self.order._approve_state == u'已提交')
        # 经理审批
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'审批中')
        # 自己先拒绝
        self.order.good_process_refused(self.order.id, self.order._name)
        # 经理重新审批
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        # 自己审批
        self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'已审批')
        self.order.buy_order_done()
        self.order.buy_order_draft()
  #      self.order.unlink()   暂时注释掉

    def test_approver_sequence(self):
        """审批顺序"""
        group_1 = self.env.ref('scm.group_process_buy_order')
        # 自己审批
        result = self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(result[0] == u'您不是这张单据的下一个审批者')

        # admin的经理改为空，将用户组1中的用户改为Alice,admin去审批
        self.staff_admin.parent_id = False  #TODO:不起作用
        group_1.write({'users': [(6, 0, [self.env.ref('scm.user_alice').id])]})
        res = self.order.good_process_approve(self.order.id, self.order._name)
        self.assertTrue(res[0] == u'您不是这张单据的下一个审批者')

    def test_refuser_sequence(self):
        """拒绝顺序"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)
        # 自己拒绝
        result = self.order.good_process_refused(self.order.id, self.order._name)
        self.assertTrue(result[0] == u'您是第一批需要审批的人，无需拒绝！')

        # 经理和自己审批之后
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.order.good_process_approve(self.order.id, self.order._name)
        res = self.order.good_process_refused(self.order.id, self.order._name)
        self.assertTrue(res[0] == u'已经通过不能拒绝！')

    def test_unlink(self):
        """级联删除"""
        self.order.unlink()

    def test_write(self):
        """write 审批异常流程"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)

        # 已提交时审核报错
        self.assertTrue(self.order._approve_state == u'已提交')
        with self.assertRaises(ValidationError):
            self.order.buy_order_done()
        # 经理审批，审批中审核报错
        self.order.with_env(env2).good_process_approve(self.order.id, self.order._name)
        self.assertTrue(self.order._approve_state == u'审批中')
        with self.assertRaises(ValidationError):
            self.order.buy_order_done()
        # 审批中修改其他字段报错，不可删除
        with self.assertRaises(ValidationError):
            self.order.date = '2017-04-19'
        with self.assertRaises(ValidationError):
            self.order.unlink()
        # 已审批不可修改,不可删除
        self.order.good_process_approve(self.order.id, self.order._name)
        with self.assertRaises(ValidationError):
            self.order.date = '2017-04-19'
        with self.assertRaises(ValidationError):
            self.order.unlink()
        # 已审核的单据不可删除
        self.order.buy_order_done()
        with self.assertRaises(UserError):
            self.order.unlink()

class test_approver(TransactionCase):

    def setUp(self):
        '''准备基本数据'''
        super(test_approver, self).setUp()
        self.approve_rule = self.browse_ref('scm.process_buy_order') # 审批规则
        self.order = self.env.ref('buy.buy_order_1').copy()
        self.user_demo = self.browse_ref('base.user_demo')

    def test_goto(self):
        """查看并处理"""
        env2 = self.env(self.env.cr, self.user_demo.id, self.env.context)
        self.order.with_env(env2)._to_approver_ids[0].goto()
<|endoftext|>


<commit_before>"""Tests for metrics."""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import itertools
import os

import numpy as np
from numpy.testing import (assert_almost_equal, assert_equal,
                           assert_raises_regex)

from scorelib.metrics import (bcubed, conditional_entropy, contingency_matrix,
                              der, jer, goodman_kruskal_tau,
                              mutual_information)
from scorelib.rttm import load_rttm
from scorelib.score import turns_to_frames

TEST_DIR = os.path.dirname(os.path.abspath(__file__))

def make_labels(n=1000, n_classes=3, one_hot=False, seed=99999):
    """Return some cluster labelings for a toy dataset."""
    rstate = np.random.RandomState(seed)
    def _one_hot(x):
        classes, x = np.unique(x, return_inverse=True)
        return np.column_stack([x == c for c in classes])
    ref_labels = rstate.randint(n_classes, size=n)
    sys_labels = rstate.randint(n_classes, size=n)
    if one_hot:
        ref_labels = _one_hot(ref_labels)
        sys_labels = _one_hot(sys_labels)
    return ref_labels, sys_labels

def test_contingency_matrix():
    # Test exceptions.
    with assert_raises_regex(
            ValueError, 'ref_labels and sys_labels should either both be 1D'):
        contingency_matrix(np.zeros(5), np.zeros((5, 2)))
    with assert_raises_regex(
            ValueError, 'ref_labels and sys_labels must have same size'):
        contingency_matrix(np.arange(5), np.arange(6))

    # Test 1-D inputs.
    X, Y = make_labels()
    cm = contingency_matrix(X, Y)
    cm_expected = np.array(
        [[106, 114, 117],
         [110, 130, 105],
         [92, 118, 108]])
    assert_equal(cm, cm_expected)

    # Test 2-D inputs.
    X, Y = make_labels(one_hot=True)
    cm = contingency_matrix(X, Y)
    assert_equal(cm, cm_expected)

def test_bcubed():
    x, y = make_labels()

    # Test from inputs.
    p, r, f1 = bcubed(x, y)
    assert_almost_equal(p, 0.3345, 3)
    assert_almost_equal(r, 0.3356, 3)
    assert_almost_equal(f1, 0.3351, 3)

    # Test from CM.
    cm = contingency_matrix(x, y)
    p, r, f1 = bcubed(None, None, cm)
    assert_almost_equal(p, 0.3345, 3)
    assert_almost_equal(r, 0.3356, 3)
    assert_almost_equal(f1, 0.3351, 3)

def test_goodman_kruskal_tau():
    x, y = make_labels()

    # Test from inputs.
    tau_rs, tau_sr = goodman_kruskal_tau(x, y)
    assert_almost_equal(tau_rs, 0.001223, 5)
    assert_almost_equal(tau_sr, 0.001223, 5)

    # Test from CM.
    cm = contingency_matrix(x, y)
    tau_rs, tau_sr = goodman_kruskal_tau(None, None, cm)
    assert_almost_equal(tau_rs, 0.001223, 5)
    assert_almost_equal(tau_sr, 0.001223, 5)

def test_conditional_entropy():
    x, y = make_labels()

    # Test from inputs.
    ce = conditional_entropy(x, y)
    assert_almost_equal(ce, 1.5824, 3)

    # Test from CM.
    cm = contingency_matrix(x, y)
    ce = conditional_entropy(None, None, cm)
    assert_almost_equal(ce, 1.5824, 3)

def test_mutual_information():
    x, y = make_labels()

    # Test from inputs.
    mi, nmi = mutual_information(x, y)
    assert_almost_equal(mi, 0.001767, 5)
    assert_almost_equal(nmi, 0.001116, 5)

    # Test from CM.
    cm = contingency_matrix(x, y)
    mi, nmi = mutual_information(None, None, cm)
    assert_almost_equal(mi, 0.001767, 5)
    assert_almost_equal(nmi, 0.001116, 5)

def test_der():
    ref_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'ref.rttm'))
    sys_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'sys.rttm'))
    expected_der = 26.3931
    file_to_der, global_der = der(ref_turns, sys_turns)
    assert_almost_equal(file_to_der['FILE1'], expected_der, 3)
    assert_almost_equal(global_der, expected_der, 3)

def test_jer():
    # Check input validation.
    with assert_raises_regex(
            ValueError, 'All passed dicts must have same'):
        jer(dict(), dict(F1=np.zeros(1)), dict())
        jer(dict(F1=np.zeros(1)), dict(), dict())
        jer(dict(), dict(), dict(F1=np.zeros(1)))

    # Edge case: no reference speech.
    ref_durs = np.array([], dtype='int64')
    sys_durs = np.array([5, 5], dtype='int64')
    cm = np.zeros((0, 2), dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 100.
    assert global_jer == 100.

    # Edge case: no system speech.
    ref_durs = np.array([5, 5], dtype='int64')
    sys_durs = np.array([], dtype='int64')
    cm = np.zeros((2, 0), dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 100.
    assert global_jer == 100.

    # Edge case: no reference OR system speech.
    ref_durs = np.array([], dtype='int64')
    sys_durs = np.array([], dtype='int64')
    cm = np.zeros([], dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 0.
    assert global_jer == 0.

    # Real data.
    ref_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'ref.rttm'))
    sys_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'sys.rttm'))
    dur = 1 + max(turn.offset
                  for turn in itertools.chain(ref_turns, sys_turns))
    ref_labels = turns_to_frames(ref_turns, [(0, dur)], step=0.01)
    sys_labels = turns_to_frames(sys_turns, [(0, dur)], step=0.01)
    cm = contingency_matrix(ref_labels, sys_labels)
    ref_durs = ref_labels.sum(axis=0)
    sys_durs = sys_labels.sum(axis=0)
    file_to_jer, global_jer = jer(
        dict(FILE1=ref_durs), dict(FILE1=sys_durs), dict(FILE1=cm))
    assert_almost_equal(file_to_jer['FILE1'], 33.24631, 3)
    assert_almost_equal(global_jer, 33.24631, 3)
<commit_msg>BUG: Correctly initialize contingency matrix for test.<commit_after>"""Tests for metrics."""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
import itertools
import os

import numpy as np
from numpy.testing import (assert_almost_equal, assert_equal,
                           assert_raises_regex)

from scorelib.metrics import (bcubed, conditional_entropy, contingency_matrix,
                              der, jer, goodman_kruskal_tau,
                              mutual_information)
from scorelib.rttm import load_rttm
from scorelib.score import turns_to_frames

TEST_DIR = os.path.dirname(os.path.abspath(__file__))

def make_labels(n=1000, n_classes=3, one_hot=False, seed=99999):
    """Return some cluster labelings for a toy dataset."""
    rstate = np.random.RandomState(seed)
    def _one_hot(x):
        classes, x = np.unique(x, return_inverse=True)
        return np.column_stack([x == c for c in classes])
    ref_labels = rstate.randint(n_classes, size=n)
    sys_labels = rstate.randint(n_classes, size=n)
    if one_hot:
        ref_labels = _one_hot(ref_labels)
        sys_labels = _one_hot(sys_labels)
    return ref_labels, sys_labels

def test_contingency_matrix():
    # Test exceptions.
    with assert_raises_regex(
            ValueError, 'ref_labels and sys_labels should either both be 1D'):
        contingency_matrix(np.zeros(5), np.zeros((5, 2)))
    with assert_raises_regex(
            ValueError, 'ref_labels and sys_labels must have same size'):
        contingency_matrix(np.arange(5), np.arange(6))

    # Test 1-D inputs.
    X, Y = make_labels()
    cm = contingency_matrix(X, Y)
    cm_expected = np.array(
        [[106, 114, 117],
         [110, 130, 105],
         [92, 118, 108]])
    assert_equal(cm, cm_expected)

    # Test 2-D inputs.
    X, Y = make_labels(one_hot=True)
    cm = contingency_matrix(X, Y)
    assert_equal(cm, cm_expected)

def test_bcubed():
    x, y = make_labels()

    # Test from inputs.
    p, r, f1 = bcubed(x, y)
    assert_almost_equal(p, 0.3345, 3)
    assert_almost_equal(r, 0.3356, 3)
    assert_almost_equal(f1, 0.3351, 3)

    # Test from CM.
    cm = contingency_matrix(x, y)
    p, r, f1 = bcubed(None, None, cm)
    assert_almost_equal(p, 0.3345, 3)
    assert_almost_equal(r, 0.3356, 3)
    assert_almost_equal(f1, 0.3351, 3)

def test_goodman_kruskal_tau():
    x, y = make_labels()

    # Test from inputs.
    tau_rs, tau_sr = goodman_kruskal_tau(x, y)
    assert_almost_equal(tau_rs, 0.001223, 5)
    assert_almost_equal(tau_sr, 0.001223, 5)

    # Test from CM.
    cm = contingency_matrix(x, y)
    tau_rs, tau_sr = goodman_kruskal_tau(None, None, cm)
    assert_almost_equal(tau_rs, 0.001223, 5)
    assert_almost_equal(tau_sr, 0.001223, 5)

def test_conditional_entropy():
    x, y = make_labels()

    # Test from inputs.
    ce = conditional_entropy(x, y)
    assert_almost_equal(ce, 1.5824, 3)

    # Test from CM.
    cm = contingency_matrix(x, y)
    ce = conditional_entropy(None, None, cm)
    assert_almost_equal(ce, 1.5824, 3)

def test_mutual_information():
    x, y = make_labels()

    # Test from inputs.
    mi, nmi = mutual_information(x, y)
    assert_almost_equal(mi, 0.001767, 5)
    assert_almost_equal(nmi, 0.001116, 5)

    # Test from CM.
    cm = contingency_matrix(x, y)
    mi, nmi = mutual_information(None, None, cm)
    assert_almost_equal(mi, 0.001767, 5)
    assert_almost_equal(nmi, 0.001116, 5)

def test_der():
    ref_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'ref.rttm'))
    sys_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'sys.rttm'))
    expected_der = 26.3931
    file_to_der, global_der = der(ref_turns, sys_turns)
    assert_almost_equal(file_to_der['FILE1'], expected_der, 3)
    assert_almost_equal(global_der, expected_der, 3)

def test_jer():
    # Check input validation.
    with assert_raises_regex(
            ValueError, 'All passed dicts must have same'):
        jer(dict(), dict(F1=np.zeros(1)), dict())
        jer(dict(F1=np.zeros(1)), dict(), dict())
        jer(dict(), dict(), dict(F1=np.zeros(1)))

    # Edge case: no reference speech.
    ref_durs = np.array([], dtype='int64')
    sys_durs = np.array([5, 5], dtype='int64')
    cm = np.zeros((0, 2), dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 100.
    assert global_jer == 100.

    # Edge case: no system speech.
    ref_durs = np.array([5, 5], dtype='int64')
    sys_durs = np.array([], dtype='int64')
    cm = np.zeros((2, 0), dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 100.
    assert global_jer == 100.

    # Edge case: no reference OR system speech.
    ref_durs = np.array([], dtype='int64')
    sys_durs = np.array([], dtype='int64')
    cm = np.zeros((0, 0), dtype='int64')
    file_to_jer, global_jer = jer(
        dict(F=ref_durs), dict(F=sys_durs), dict(F=cm))
    assert file_to_jer['F'] == 0.
    assert global_jer == 0.

    # Real data.
    ref_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'ref.rttm'))
    sys_turns, _, _ = load_rttm(
        os.path.join(TEST_DIR, 'sys.rttm'))
    dur = 1 + max(turn.offset
                  for turn in itertools.chain(ref_turns, sys_turns))
    ref_labels = turns_to_frames(ref_turns, [(0, dur)], step=0.01)
    sys_labels = turns_to_frames(sys_turns, [(0, dur)], step=0.01)
    cm = contingency_matrix(ref_labels, sys_labels)
    ref_durs = ref_labels.sum(axis=0)
    sys_durs = sys_labels.sum(axis=0)
    file_to_jer, global_jer = jer(
        dict(FILE1=ref_durs), dict(FILE1=sys_durs), dict(FILE1=cm))
    assert_almost_equal(file_to_jer['FILE1'], 33.24631, 3)
    assert_almost_equal(global_jer, 33.24631, 3)
<|endoftext|>