<commit_before>from copy import copy

from sfepy.base.base import output, get_default, OneTypeList, Struct,\
     get_default_attr
from sfepy.terms import DataCaches
from sfepy.fem import Equations, Variables, Region
from sfepy.fem.fields import setup_dof_conns, setup_extra_data

##
# 02.10.2007, c
class Evaluator( Struct ):
    pass

##
# 02.10.2007, c
class BasicEvaluator( Evaluator ):

    def __init__(self, problem, matrix_hook=None):
        Evaluator.__init__(self, problem=problem,
                           matrix_hook=matrix_hook)

    def new_ulf_iteration(self, nls, vec, it, err, err0):

        pb = self.problem

        vec = self.make_full_vec(vec)
        pb.equations.set_variables_from_state(vec)

        upd_vars = get_default_attr(pb.conf.options, 'mesh_update_variables', None)
        for varname in upd_vars:
            try:
                state = pb.equations.variables[varname]
            except IndexError:
                msg = 'variable "%s" does not exist!' % varname
                raise KeyError( msg )

        nods = state.field.get_dofs_in_region(state.field.region, merge=True)
        coors = pb.domain.get_mesh_coors().copy()
        vs = state()
        coors[nods,:] = coors[nods,:] + vs.reshape(len(nods), state.n_components)
        if pb.ts.step == 1 and it == 0:
            pb.equations.clear_geometries(save=True)
        else:
            pb.equations.clear_geometries()

        pb.set_mesh_coors(coors, update_fields=True, actual=True)

    def eval_residual( self, vec, is_full = False ):
        if not is_full:
            vec = self.make_full_vec( vec )
        try:
            pb = self.problem
            vec_r = pb.equations.eval_residuals(vec)

        except StopIteration, exc:
            status = exc.args[0]
            output( 'error %d in term "%s" of equation "%s"!'\
                    % (status, exc.args[1].name, exc.args[2].desc) )
            raise ValueError

        return vec_r

    def eval_tangent_matrix( self, vec, mtx = None, is_full = False ):
        if isinstance( vec, str ) and vec == 'linear':
            return get_default(mtx, self.problem.mtx_a)

        if not is_full:
            vec = self.make_full_vec( vec )
        try:
            pb = self.problem
            if mtx is None:
                mtx = pb.mtx_a
            mtx = pb.equations.eval_tangent_matrices(vec, mtx)

        except StopIteration, exc:
            status = exc.args[0]
            output( ('error %d in term "%s" of derivative of equation "%s"'
                     + ' with respect to variable "%s"!')\
                    % (status,
                       exc.args[1].name, exc.args[2].desc, exc.args[3] ) )
            raise ValueError

        if self.matrix_hook is not None:
            mtx = self.matrix_hook(mtx, self.problem, call_mode='basic')

        return mtx

    def make_full_vec( self, vec ):
        return self.problem.equations.make_full_vec(vec)

##
# 04.10.2007, c
class LCBCEvaluator( BasicEvaluator ):

    ##
    # 04.10.2007, c
    def __init__(self, problem, matrix_hook=None):
        BasicEvaluator.__init__(self, problem, matrix_hook=matrix_hook)
        self.op_lcbc = problem.equations.get_lcbc_operator()

    ##
    # 04.10.2007, c
    def eval_residual( self, vec, is_full = False ):
        if not is_full:
            vec = self.make_full_vec( vec )
        vec_r = BasicEvaluator.eval_residual( self, vec, is_full = True )
        vec_rr = self.op_lcbc.T * vec_r
        return vec_rr

    ##
    # 04.10.2007, c
    def eval_tangent_matrix( self, vec, mtx = None, is_full = False ):
        if isinstance( vec, str ) and vec == 'linear':
            return get_default(mtx, self.problem.mtx_a)

        if not is_full:
            vec = self.make_full_vec( vec )
        mtx = BasicEvaluator.eval_tangent_matrix( self, vec, mtx = mtx,
                                                  is_full = True )
        mtx_r = self.op_lcbc.T * mtx * self.op_lcbc
        mtx_r = mtx_r.tocsr()
        mtx_r.sort_indices()
##         import pylab
##         from sfepy.base.plotutils import spy
##         spy( mtx_r )
##         pylab.show()
##         print mtx_r.__repr__()

        if self.matrix_hook is not None:
            mtx_r = self.matrix_hook(mtx_r, self.problem, call_mode='lcbc')

        return mtx_r

def create_evaluable(expression, fields, materials, variables, integrals,
                     regions=None,
                     ebcs=None, epbcs=None, lcbcs=None, ts=None, functions=None,
                     auto_init=False, mode='eval', extra_args=None,
                     verbose=True, kwargs=None):
    """
    Create evaluable object (equations and corresponding variables)
    from the `expression` string.

    Parameters
    ----------
    expression : str
        The expression to evaluate.
    fields : dict
        The dictionary of fields used in `variables`.
    materials : Materials instance
        The materials used in the expression.
    variables : Variables instance
        The variables used in the expression.
    integrals : Integrals instance
        The integrals to be used.
    regions : Region instance or list of Region instances
        The region(s) to be used. If not given, the regions defined
        within the fields domain are used.
    ebcs : Conditions instance, optional
        The essential (Dirichlet) boundary conditions for 'weak'
        mode.
    epbcs : Conditions instance, optional
        The periodic boundary conditions for 'weak'
        mode.
    lcbcs : Conditions instance, optional
        The linear combination boundary conditions for 'weak'
        mode.
    ts : TimeStepper instance, optional
        The time stepper.
    functions : Functions instance, optional
        The user functions for boundary conditions, materials
        etc.
    auto_init : bool
        Set values of all variables to all zeros.
    mode : one of 'eval', 'el_avg', 'qp', 'weak'
        The evaluation mode - 'weak' means the finite element
        assembling, 'qp' requests the values in quadrature points,
        'el_avg' element averages and 'eval' means integration over
        each term region.
    extra_args : dict, optional
        Extra arguments to be passed to terms in the expression.
    verbose : bool
        If False, reduce verbosity.
    kwargs : dict, optional
        The variables (dictionary of (variable name) : (Variable
        instance)) to be used in the expression.

    Returns
    -------
    equation : Equation instance
        The equation that is ready to be evaluated.
    variables : Variables instance
        The variables used in the equation.
    """
    if kwargs is None:
        kwargs = {}

    if regions is not None:
        if isinstance(regions, Region):
            regions = [regions]

        regions = OneTypeList(Region, regions)

    else:
        regions = fields[fields.keys()[0]].domain.regions

    caches = DataCaches()

    # Create temporary variables.
    aux_vars = Variables(variables)

    if extra_args is None:
        extra_args = kwargs

    else:
        extra_args = copy(extra_args)
        extra_args.update(kwargs)

    equations = Equations.from_conf({'tmp' : expression},
                                    aux_vars, regions, materials, integrals,
                                    setup=False,
                                    caches=caches, user=extra_args,
                                    verbose=verbose)
    equations.collect_conn_info()

    # The true variables used in the expression.
    variables = equations.variables
    if auto_init:
        for var in variables:
            var.init_data(step=0)

    if mode == 'weak':
        setup_dof_conns(equations.conn_info)
        equations.time_update(ts, ebcs, epbcs, lcbcs, functions)

    else:
        setup_extra_data(equations.conn_info)

    return equations, variables

def eval_equations(equations, variables, clear_caches=True,
                   mode='eval', dw_mode='vector', term_mode=None):
    """
    Evaluate the equations.

    Parameters
    ----------
    equations : Equations instance
        The equations returned by :func:`create_evaluable()`.
    variables : Variables instance
        The variables returned by :func:`create_evaluable()`.
    clear_caches : bool
        If True, clear term caches.
    mode : one of 'eval', 'el_avg', 'qp', 'weak'
        The evaluation mode - 'weak' means the finite element
        assembling, 'qp' requests the values in quadrature points,
        'el_avg' element averages and 'eval' means integration over
        each term region.
    dw_mode : 'vector' or 'matrix'
        The assembling mode for 'weak' evaluation mode.
    term_mode : str
        The term call mode - some terms support different call modes
        and depending on the call mode different values are
        returned.

    Returns
    -------
    out : array
        The result of the evaluation.
    """
    asm_obj = None

    if mode == 'weak':
        if dw_mode == 'vector':
            asm_obj = equations.create_stripped_state_vector()

        else:
            asm_obj = equations.create_matrix_graph()

    if clear_caches:
        equations.invalidate_term_caches()

    out = equations.evaluate(mode=mode, dw_mode=dw_mode, term_mode=term_mode,
                             asm_obj=asm_obj)

    if variables.has_lcbc and mode == 'weak':
        op_lcbc = variables.op_lcbc
        if dw_mode == 'vector':
            out = op_lcbc.T * out

        elif dw_mode == 'matrix':
            out = op_lcbc.T * out * op_lcbc
            out = out.tocsr()
            out.sort_indices()

    return out

def evaluate(expression, fields, materials, variables, integrals,
             update_materials=True,
             ebcs=None, epbcs=None, lcbcs=None, ts=None, functions=None,
             auto_init=False, mode='eval', dw_mode='vector', term_mode=None,
             ret_variables=False, extra_args=None, verbose=True, kwargs=None):
    """
    Convenience function calling :func:`create_evaluable()` and
    :func:`eval_equations()`.

    Parameters
    ----------
    ... : arguments
        See docstrings of :func:`create_evaluable()` and
        :func:`eval_equations()`.
    """
    aux = create_evaluable(expression, fields, materials, variables, integrals,
                           update_materials=update_materials,
                           ebcs=ebcs, epbcs=epbcs, lcbcs=lcbcs, ts=ts,
                           functions=functions, auto_init=auto_init,
                           mode=mode, extra_args=extra_args,
                           verbose=verbose, kwargs=kwargs)
    equations, variables = aux

    out = eval_equations(equations, variables,
                         mode=mode, dw_mode=dw_mode, term_mode=term_mode)

    if ret_variables:
        out = (out, variables)

    return out

def assemble_by_blocks(conf_equations, problem, ebcs=None, epbcs=None,
                       dw_mode='matrix'):
    """Instead of a global matrix, return its building blocks as defined in
    `conf_equations`. The name and row/column variables of each block have to
    be encoded in the equation's name, as in:

    conf_equations = {
      'A,v,u' : "dw_lin_elastic_iso.i1.Y2( inclusion.lame, v, u )",
    }

    Notes
    -----
    `ebcs`, `epbcs` must be either lists of BC names, or BC configuration
    dictionaries.
    """
    if isinstance( ebcs, list ) and isinstance( epbcs, list ):
        bc_mode = 0
    elif isinstance( ebcs, dict ) and isinstance( epbcs, dict ):
        bc_mode = 1
    else:
        raise TypeError('bad BC!')

    matrices = {}
    for key, mtx_term in conf_equations.iteritems():
        ks = key.split( ',' )
        mtx_name, var_names = ks[0], ks[1:]
        output( mtx_name, var_names )

        problem.set_equations({'eq': mtx_term})
        variables = problem.get_variables()
        indx = variables.get_indx

        if bc_mode == 0:
            problem.select_bcs( ebc_names = ebcs, epbc_names = epbcs )

        else:
            problem.time_update(ebcs=ebcs, epbcs=epbcs)

        ir = indx( var_names[0], stripped = True, allow_dual = True )
        ic = indx( var_names[1], stripped = True, allow_dual = True )

        problem.update_materials()
        mtx = problem.evaluate(mtx_term, auto_init=True,
                               mode='weak', dw_mode='matrix',
                               copy_materials=False)
        matrices[mtx_name] = mtx[ir,ic]

    return matrices
<commit_msg>fix evaluate.new_ulf_iteration() - updated Lagrangian formulation<commit_after>from copy import copy

from sfepy.base.base import output, get_default, OneTypeList, Struct,\
     get_default_attr
from sfepy.terms import DataCaches
from sfepy.fem import Equations, Variables, Region
from sfepy.fem.fields import setup_dof_conns, setup_extra_data

##
# 02.10.2007, c
class Evaluator( Struct ):
    pass

##
# 02.10.2007, c
class BasicEvaluator( Evaluator ):

    def __init__(self, problem, matrix_hook=None):
        Evaluator.__init__(self, problem=problem,
                           matrix_hook=matrix_hook)

    def new_ulf_iteration(self, nls, vec, it, err, err0):

        pb = self.problem

        vec = self.make_full_vec(vec)
        pb.equations.set_variables_from_state(vec)

        upd_vars = get_default_attr(pb.conf.options, 'mesh_update_variables', None)
        for varname in upd_vars:
            try:
                state = pb.equations.variables[varname]
            except IndexError:
                msg = 'variable "%s" does not exist!' % varname
                raise KeyError( msg )

        nods = state.field.get_dofs_in_region(state.field.region, merge=True)
        coors = pb.domain.get_mesh_coors().copy()
        vs = state()
        coors[nods,:] = coors[nods,:] + vs.reshape(len(nods), state.n_components)
        if pb.ts.step == 1 and it == 0:
            state.field.save_mappings()
        else:
            state.field.clear_mappings()
        pb.set_mesh_coors(coors, update_fields=False, actual=True,
                          clear_all=False)

    def eval_residual( self, vec, is_full = False ):
        if not is_full:
            vec = self.make_full_vec( vec )
        try:
            pb = self.problem
            vec_r = pb.equations.eval_residuals(vec)

        except StopIteration, exc:
            status = exc.args[0]
            output( 'error %d in term "%s" of equation "%s"!'\
                    % (status, exc.args[1].name, exc.args[2].desc) )
            raise ValueError

        return vec_r

    def eval_tangent_matrix( self, vec, mtx = None, is_full = False ):
        if isinstance( vec, str ) and vec == 'linear':
            return get_default(mtx, self.problem.mtx_a)

        if not is_full:
            vec = self.make_full_vec( vec )
        try:
            pb = self.problem
            if mtx is None:
                mtx = pb.mtx_a
            mtx = pb.equations.eval_tangent_matrices(vec, mtx)

        except StopIteration, exc:
            status = exc.args[0]
            output( ('error %d in term "%s" of derivative of equation "%s"'
                     + ' with respect to variable "%s"!')\
                    % (status,
                       exc.args[1].name, exc.args[2].desc, exc.args[3] ) )
            raise ValueError

        if self.matrix_hook is not None:
            mtx = self.matrix_hook(mtx, self.problem, call_mode='basic')

        return mtx

    def make_full_vec( self, vec ):
        return self.problem.equations.make_full_vec(vec)

##
# 04.10.2007, c
class LCBCEvaluator( BasicEvaluator ):

    ##
    # 04.10.2007, c
    def __init__(self, problem, matrix_hook=None):
        BasicEvaluator.__init__(self, problem, matrix_hook=matrix_hook)
        self.op_lcbc = problem.equations.get_lcbc_operator()

    ##
    # 04.10.2007, c
    def eval_residual( self, vec, is_full = False ):
        if not is_full:
            vec = self.make_full_vec( vec )
        vec_r = BasicEvaluator.eval_residual( self, vec, is_full = True )
        vec_rr = self.op_lcbc.T * vec_r
        return vec_rr

    ##
    # 04.10.2007, c
    def eval_tangent_matrix( self, vec, mtx = None, is_full = False ):
        if isinstance( vec, str ) and vec == 'linear':
            return get_default(mtx, self.problem.mtx_a)

        if not is_full:
            vec = self.make_full_vec( vec )
        mtx = BasicEvaluator.eval_tangent_matrix( self, vec, mtx = mtx,
                                                  is_full = True )
        mtx_r = self.op_lcbc.T * mtx * self.op_lcbc
        mtx_r = mtx_r.tocsr()
        mtx_r.sort_indices()
##         import pylab
##         from sfepy.base.plotutils import spy
##         spy( mtx_r )
##         pylab.show()
##         print mtx_r.__repr__()

        if self.matrix_hook is not None:
            mtx_r = self.matrix_hook(mtx_r, self.problem, call_mode='lcbc')

        return mtx_r

def create_evaluable(expression, fields, materials, variables, integrals,
                     regions=None,
                     ebcs=None, epbcs=None, lcbcs=None, ts=None, functions=None,
                     auto_init=False, mode='eval', extra_args=None,
                     verbose=True, kwargs=None):
    """
    Create evaluable object (equations and corresponding variables)
    from the `expression` string.

    Parameters
    ----------
    expression : str
        The expression to evaluate.
    fields : dict
        The dictionary of fields used in `variables`.
    materials : Materials instance
        The materials used in the expression.
    variables : Variables instance
        The variables used in the expression.
    integrals : Integrals instance
        The integrals to be used.
    regions : Region instance or list of Region instances
        The region(s) to be used. If not given, the regions defined
        within the fields domain are used.
    ebcs : Conditions instance, optional
        The essential (Dirichlet) boundary conditions for 'weak'
        mode.
    epbcs : Conditions instance, optional
        The periodic boundary conditions for 'weak'
        mode.
    lcbcs : Conditions instance, optional
        The linear combination boundary conditions for 'weak'
        mode.
    ts : TimeStepper instance, optional
        The time stepper.
    functions : Functions instance, optional
        The user functions for boundary conditions, materials
        etc.
    auto_init : bool
        Set values of all variables to all zeros.
    mode : one of 'eval', 'el_avg', 'qp', 'weak'
        The evaluation mode - 'weak' means the finite element
        assembling, 'qp' requests the values in quadrature points,
        'el_avg' element averages and 'eval' means integration over
        each term region.
    extra_args : dict, optional
        Extra arguments to be passed to terms in the expression.
    verbose : bool
        If False, reduce verbosity.
    kwargs : dict, optional
        The variables (dictionary of (variable name) : (Variable
        instance)) to be used in the expression.

    Returns
    -------
    equation : Equation instance
        The equation that is ready to be evaluated.
    variables : Variables instance
        The variables used in the equation.
    """
    if kwargs is None:
        kwargs = {}

    if regions is not None:
        if isinstance(regions, Region):
            regions = [regions]

        regions = OneTypeList(Region, regions)

    else:
        regions = fields[fields.keys()[0]].domain.regions

    caches = DataCaches()

    # Create temporary variables.
    aux_vars = Variables(variables)

    if extra_args is None:
        extra_args = kwargs

    else:
        extra_args = copy(extra_args)
        extra_args.update(kwargs)

    equations = Equations.from_conf({'tmp' : expression},
                                    aux_vars, regions, materials, integrals,
                                    setup=False,
                                    caches=caches, user=extra_args,
                                    verbose=verbose)
    equations.collect_conn_info()

    # The true variables used in the expression.
    variables = equations.variables
    if auto_init:
        for var in variables:
            var.init_data(step=0)

    if mode == 'weak':
        setup_dof_conns(equations.conn_info)
        equations.time_update(ts, ebcs, epbcs, lcbcs, functions)

    else:
        setup_extra_data(equations.conn_info)

    return equations, variables

def eval_equations(equations, variables, clear_caches=True,
                   mode='eval', dw_mode='vector', term_mode=None):
    """
    Evaluate the equations.

    Parameters
    ----------
    equations : Equations instance
        The equations returned by :func:`create_evaluable()`.
    variables : Variables instance
        The variables returned by :func:`create_evaluable()`.
    clear_caches : bool
        If True, clear term caches.
    mode : one of 'eval', 'el_avg', 'qp', 'weak'
        The evaluation mode - 'weak' means the finite element
        assembling, 'qp' requests the values in quadrature points,
        'el_avg' element averages and 'eval' means integration over
        each term region.
    dw_mode : 'vector' or 'matrix'
        The assembling mode for 'weak' evaluation mode.
    term_mode : str
        The term call mode - some terms support different call modes
        and depending on the call mode different values are
        returned.

    Returns
    -------
    out : array
        The result of the evaluation.
    """
    asm_obj = None

    if mode == 'weak':
        if dw_mode == 'vector':
            asm_obj = equations.create_stripped_state_vector()

        else:
            asm_obj = equations.create_matrix_graph()

    if clear_caches:
        equations.invalidate_term_caches()

    out = equations.evaluate(mode=mode, dw_mode=dw_mode, term_mode=term_mode,
                             asm_obj=asm_obj)

    if variables.has_lcbc and mode == 'weak':
        op_lcbc = variables.op_lcbc
        if dw_mode == 'vector':
            out = op_lcbc.T * out

        elif dw_mode == 'matrix':
            out = op_lcbc.T * out * op_lcbc
            out = out.tocsr()
            out.sort_indices()

    return out

def evaluate(expression, fields, materials, variables, integrals,
             update_materials=True,
             ebcs=None, epbcs=None, lcbcs=None, ts=None, functions=None,
             auto_init=False, mode='eval', dw_mode='vector', term_mode=None,
             ret_variables=False, extra_args=None, verbose=True, kwargs=None):
    """
    Convenience function calling :func:`create_evaluable()` and
    :func:`eval_equations()`.

    Parameters
    ----------
    ... : arguments
        See docstrings of :func:`create_evaluable()` and
        :func:`eval_equations()`.
    """
    aux = create_evaluable(expression, fields, materials, variables, integrals,
                           update_materials=update_materials,
                           ebcs=ebcs, epbcs=epbcs, lcbcs=lcbcs, ts=ts,
                           functions=functions, auto_init=auto_init,
                           mode=mode, extra_args=extra_args,
                           verbose=verbose, kwargs=kwargs)
    equations, variables = aux

    out = eval_equations(equations, variables,
                         mode=mode, dw_mode=dw_mode, term_mode=term_mode)

    if ret_variables:
        out = (out, variables)

    return out

def assemble_by_blocks(conf_equations, problem, ebcs=None, epbcs=None,
                       dw_mode='matrix'):
    """Instead of a global matrix, return its building blocks as defined in
    `conf_equations`. The name and row/column variables of each block have to
    be encoded in the equation's name, as in:

    conf_equations = {
      'A,v,u' : "dw_lin_elastic_iso.i1.Y2( inclusion.lame, v, u )",
    }

    Notes
    -----
    `ebcs`, `epbcs` must be either lists of BC names, or BC configuration
    dictionaries.
    """
    if isinstance( ebcs, list ) and isinstance( epbcs, list ):
        bc_mode = 0
    elif isinstance( ebcs, dict ) and isinstance( epbcs, dict ):
        bc_mode = 1
    else:
        raise TypeError('bad BC!')

    matrices = {}
    for key, mtx_term in conf_equations.iteritems():
        ks = key.split( ',' )
        mtx_name, var_names = ks[0], ks[1:]
        output( mtx_name, var_names )

        problem.set_equations({'eq': mtx_term})
        variables = problem.get_variables()
        indx = variables.get_indx

        if bc_mode == 0:
            problem.select_bcs( ebc_names = ebcs, epbc_names = epbcs )

        else:
            problem.time_update(ebcs=ebcs, epbcs=epbcs)

        ir = indx( var_names[0], stripped = True, allow_dual = True )
        ic = indx( var_names[1], stripped = True, allow_dual = True )

        problem.update_materials()
        mtx = problem.evaluate(mtx_term, auto_init=True,
                               mode='weak', dw_mode='matrix',
                               copy_materials=False)
        matrices[mtx_name] = mtx[ir,ic]

    return matrices
<|endoftext|>


<commit_before># Use workbook fixture from BDD tests (including elasticsearch)
from .features.conftest import app_settings, app, workbook

def test_report_download(testapp, workbook):
    res = testapp.get('/report.tsv?type=Experiment&sort=accession')
    assert res.headers['content-type'] == 'text/tsv; charset=UTF-8'
    disposition = res.headers['content-disposition']
    assert disposition == 'attachment;filename="report.tsv"'
    lines = res.body.splitlines()
    assert lines[0].split(b'\t') == [
        b'ID', b'Accession', b'Assay Type', b'Assay Nickname', b'Target label',
        b'Target gene', b'Biosample', b'Description', b'Lab', b'Project', b'Status',
        b'Linked Antibody', b'Species', b'Life stage', b'Age', b'Age Units',
        b'Treatment', b'Term ID', b'Concentration', b'Concentration units',
        b'Duration', b'Duration units', b'Synchronization',
        b'Post-synchronization time', b'Post-synchronization time units',
        b'Replicates', b'Files', b'Dbxrefs'
    ]
    # Sorting for scan and limit=all is disabled currently
    # assert lines[1].split(b'\t') == [
    #     b'/experiments/ENCSR000AAL/', b'ENCSR000AAL', b'RNA-seq', b'RNA-seq',
    #     b'', b'K562', b'RNA Evaluation K562 Small Total RNA-seq from Gingeras',
    #     b'Thomas Gingeras, CSHL', b'ENCODE', b'released', b'',
    #     b'', b'', b'', b'', b'', b'', b'', b'', b'',
    #     b'', b'', b'', b'', b'', b'', b''
    # ]
    assert len(lines) == 44
<commit_msg>Adjust report download test for schema changes<commit_after># Use workbook fixture from BDD tests (including elasticsearch)
from .features.conftest import app_settings, app, workbook

def test_report_download(testapp, workbook):
    res = testapp.get('/report.tsv?type=Experiment&sort=accession')
    assert res.headers['content-type'] == 'text/tsv; charset=UTF-8'
    disposition = res.headers['content-disposition']
    assert disposition == 'attachment;filename="report.tsv"'
    lines = res.body.splitlines()
    assert lines[0].split(b'\t') == [
        b'ID', b'Accession', b'Assay Type', b'Assay Nickname', b'Target label',
        b'Target gene', b'Biosample', b'Description', b'Lab', b'Project', b'Status',
        b'Biological replicate', b'Technical replicate', b'Linked Antibody',
        b'Species', b'Life stage', b'Age', b'Age Units', b'Treatment', b'Term ID',
        b'Concentration', b'Concentration units', b'Duration', b'Duration units',
        b'Synchronization', b'Post-synchronization time',
        b'Post-synchronization time units', b'Replicates', b'Files'
    ]
    # Sorting for scan and limit=all is disabled currently
    # assert lines[1].split(b'\t') == [
    #     b'/experiments/ENCSR000AAL/', b'ENCSR000AAL', b'RNA-seq', b'RNA-seq',
    #     b'', b'K562', b'RNA Evaluation K562 Small Total RNA-seq from Gingeras',
    #     b'Thomas Gingeras, CSHL', b'ENCODE', b'released', b'',
    #     b'', b'', b'', b'', b'', b'', b'', b'', b'',
    #     b'', b'', b'', b'', b'', b'', b''
    # ]
    assert len(lines) == 44
<|endoftext|>


<commit_before>import pytest
pytestmark = pytest.mark.working

targets = [
    {'name': 'one', 'uuid': '775795d3-4410-4114-836b-8eeecf1d0c2f'},
    {'name': 'two', 'uuid': 'd6784f5e-48a1-4b40-9b11-c8aefb6e1377'},
]

item = {
    'required': 'required value',
}

simple1 = {
    'required': 'required value',
    'simple1': 'supplied simple1',
}

simple2 = {
    'required': 'required value',
    'simple2': 'supplied simple2',
}

item_with_uuid = [
    {
        'uuid': '0f13ff76-c559-4e70-9497-a6130841df9f',
        'required': 'required value 1',
        'field_no_default': 'test'
    },
    {
        'uuid': '6c3e444b-f290-43c4-bfb9-d20135377770',
        'required': 'required value 2',
    },
]

item_with_link = [
    {
        'required': 'required value 1',
        'protected_link': '775795d3-4410-4114-836b-8eeecf1d0c2f',
    },
    {
        'required': 'required value 2',
        'protected_link': 'd6784f5e-48a1-4b40-9b11-c8aefb6e1377',
    },
]

COLLECTION_URL = '/testing-post-put-patch/'

@pytest.fixture
def link_targets(testapp):
    url = '/testing-link-targets/'
    for item in targets:
        testapp.post_json(url, item, status=201)

@pytest.fixture
def content(testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=201)
    return {'@id': res.location}

@pytest.fixture
def content_with_child(testapp):
    parent_res = testapp.post_json('/testing-link-targets/', {}, status=201)
    parent_id = parent_res.json['@graph'][0]['@id']
    child_res = testapp.post_json('/testing-link-sources/', {'target': parent_id})
    child_id = child_res.json['@graph'][0]['@id']
    return {'@id': parent_id, 'child': child_id}

def test_admin_post(testapp):
    testapp.post_json(COLLECTION_URL, item, status=201)
    testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=201)

def test_submitter_post(submitter_testapp):
    testapp = submitter_testapp
    testapp.post_json(COLLECTION_URL, item, status=201)
    res = testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=422)
    assert any(error.get('name') == ['uuid'] for error in res.json['errors'])

def test_admin_put_uuid(content, testapp):
    url = content['@id']
    # so long as the same uuid is supplied, PUTing the uuid is fine
    testapp.put_json(url, item_with_uuid[0], status=200)
    # but the uuid may not be changed on PUT;
    testapp.put_json(url, item_with_uuid[1], status=422)

def test_submitter_put_uuid(content, submitter_testapp):
    testapp = submitter_testapp
    url = content['@id']
    # so long as the same uuid is supplied, PUTing the uuid is fine
    testapp.put_json(url, item_with_uuid[0], status=200)
    # but the uuid may not be changed on PUT;
    testapp.put_json(url, item_with_uuid[1], status=422)

def test_defaults_on_put(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'

    res = testapp.put_json(url, simple1, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.put_json(url, simple2, status=200)
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'
    assert res.json['@graph'][0]['simple2'] == 'supplied simple2'

def test_patch(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {'simple1': 'supplied simple1'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {'simple2': 'supplied simple2'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'supplied simple2'

def test_patch_new_schema_version(content, root, testapp, monkeypatch):
    collection = root['testing_post_put_patch']
    properties = collection.type_info.schema['properties']

    url = content['@id']
    res = testapp.get(url)
    assert res.json['schema_version'] == '1'

    monkeypatch.setitem(properties['schema_version'], 'default', '2')
    monkeypatch.setattr(collection.type_info, 'schema_version', '2')
    monkeypatch.setitem(properties, 'new_property', {'default': 'new'})
    res = testapp.patch_json(url, {}, status=200)
    assert res.json['@graph'][0]['schema_version'] == '2'
    assert res.json['@graph'][0]['new_property'] == 'new'

def test_admin_put_protected_link(link_targets, testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location

    testapp.put_json(url, item_with_link[0], status=200)
    testapp.put_json(url, item_with_link[1], status=200)

def test_submitter_put_protected_link(link_targets, testapp, submitter_testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location

    submitter_testapp.put_json(url, item_with_link[0], status=200)
    submitter_testapp.put_json(url, item_with_link[1], status=422)

def test_put_object_not_touching_children(content_with_child, testapp):
    """
    The rev_link style of editing children is removed as of FF-1089
    This will still pass
    """
    url = content_with_child['@id']
    res = testapp.put_json(url, {}, status=200)
    assert content_with_child['child'] in res.json['@graph'][0]['reverse']

def test_put_object_editing_child_does_not_work(content_with_child, testapp):
    edit = {
        'reverse': [{
            '@id': content_with_child['child'],
            'status': 'released',
        }]
    }
    testapp.put_json(content_with_child['@id'], edit, status=200)
    res = testapp.get(content_with_child['child'] + '?frame=embedded')
    assert 'status' not in res.json

def test_put_object_adding_child(content_with_child, testapp):
    """
    This works because it simply patches the reverse field, which is
    an array
    """
    edit = {
        'reverse': [
            content_with_child['child'],
            {
                'status': 'released',
            }
        ]
    }
    testapp.put_json(content_with_child['@id'], edit, status=200)
    res = testapp.get(content_with_child['@id'])
    assert len(res.json['reverse']) == 2

def test_post_object_with_child(testapp):
    """
    This functionality has changed, as reverse links are no longer supported
    This will simply post the value below to the 'reverse' field
    """
    edit = {
        'reverse': [{
            'status': 'released',
        }]
    }
    res = testapp.post_json('/testing-link-targets', edit, status=201)
    assert res.json['@graph'][0]['reverse'] == edit['reverse']

def test_retry(testapp):
    res = testapp.post_json('/testing-post-put-patch/', {'required': ''})
    url = res.location
    res = testapp.get(url + '/@@testing-retry?datstore=database')
    assert res.json['attempt'] == 2
    assert not res.json['detached']

def test_post_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    #if we post this data it will fail with uuid conflict, as calling the human fixture posts it
    testapp.post_json('/organism/', human_data, status=409)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    rest = testapp.post_json('/organism/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_put_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    #if we post this data it will fail with invalid status

    testapp.post_json('/organism/', human_data, status=409)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    rest = testapp.post_json('/organism/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_post_check_only_invalid_data(testapp, human_data):
    '''
    note theese test should work on any object
    '''
    human_data['taxon_id'] = 24;
    testapp.post_json('/organism/?check_only=True', human_data, status=422)

def test_put_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    # human_data has already been posted, now put with invalid status
    human_data['status'] = 'no a valid status'
    testapp.put_json('/organisms/human/?check_only=True', human_data, status=422)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    human_data['status'] = human['status']
    rest = testapp.put_json('/organisms/human/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_patch_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    # human_data has already been posted, now put with invalid status
    human_data['status'] = 'no a valid status'
    testapp.patch_json('/organisms/human/?check_only=True', human_data, status=422)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    human_data['status'] = human['status']
    rest = testapp.patch_json('/organisms/human/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_patch_delete_fields(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'

    res = testapp.patch_json(url, {'simple1': 'this is a test'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'this is a test'

    # delete fields with defaults resets to default, while deleting non default field
    # completely removes them
    res = testapp.patch_json(url + "?delete_fields=simple1,field_no_default", {}, status=200)
    assert 'field_no_default' not in res.json['@graph'][0].keys()
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'

def test_patch_delete_fields_non_string(content, testapp):
    url = content['@id']
    res = testapp.get(url)

    # delete fields with defaults resets to default, while deleting non default field
    # completely removes them
    res = testapp.patch_json(url + "?delete_fields=schema_version", {}, status=200)
    assert res.json['@graph'][0]['schema_version'] == '1'

def test_patch_delete_fields_still_works_with_no_validation(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'

    # with validate=false, then defaults are not populated so default fields are also deleted
    res = testapp.patch_json(url + "?delete_fields=simple1,field_no_default&validate=false", {}, status=200)
    assert 'field_no_default' not in res.json['@graph'][0].keys()
    assert 'simple1' not in res.json['@graph'][0].keys()

def test_patch_delete_fields_bad_param(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'
    res = testapp.patch_json(url + "?delete_fields=simple1,bad_fieldname", {}, status=422)
    assert res.json['description'] == "Failed validation"
    assert res.json['errors'][0]['description'] == "Additional properties are not allowed ('bad_fieldname' was unexpected)"

def test_patch_delete_fields_import_items_admin(link_targets, testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location
    assert res.json['@graph'][0]['protected_link']
    res = testapp.patch_json(url + "?delete_fields=protected_link", {}, status=200)

def test_patch_delete_fields_import_items_submitter(content, submitter_testapp):
    testapp = submitter_testapp
    url = content['@id']
    res = testapp.get(url)
    assert res.json['protected']
    res = testapp.patch_json(url + "?delete_fields=protected", {}, status=422)

def test_patch_delete_fields_required(content, testapp):
    url = content['@id']
    res = testapp.get(url)

    # with validate=false, then defaults are not populated so default fields are also deleted
    res = testapp.patch_json(url + "?delete_fields=required", {}, status=422)
    assert res.json['description'] == "Failed validation"
    assert res.json['errors'][0]['description'] == "'required' is a required property"
<commit_msg>Placeholder fixes for test_post_put_patch.py<commit_after>import pytest
pytestmark = pytest.mark.working

targets = [
    {'name': 'one', 'uuid': '775795d3-4410-4114-836b-8eeecf1d0c2f'},
    {'name': 'two', 'uuid': 'd6784f5e-48a1-4b40-9b11-c8aefb6e1377'},
]

item = {
    'required': 'required value',
}

simple1 = {
    'required': 'required value',
    'simple1': 'supplied simple1',
}

simple2 = {
    'required': 'required value',
    'simple2': 'supplied simple2',
}

item_with_uuid = [
    {
        'uuid': '0f13ff76-c559-4e70-9497-a6130841df9f',
        'required': 'required value 1',
        'field_no_default': 'test'
    },
    {
        'uuid': '6c3e444b-f290-43c4-bfb9-d20135377770',
        'required': 'required value 2',
    },
]

item_with_link = [
    {
        'required': 'required value 1',
        'protected_link': '775795d3-4410-4114-836b-8eeecf1d0c2f',
    },
    {
        'required': 'required value 2',
        'protected_link': 'd6784f5e-48a1-4b40-9b11-c8aefb6e1377',
    },
]

COLLECTION_URL = '/testing-post-put-patch/'

@pytest.fixture
def link_targets(testapp):
    url = '/testing-link-targets/'
    for item in targets:
        testapp.post_json(url, item, status=201)

@pytest.fixture
def content(testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=201)
    return {'@id': res.location}

@pytest.fixture
def content_with_child(testapp):
    parent_res = testapp.post_json('/testing-link-targets/', {}, status=201)
    parent_id = parent_res.json['@graph'][0]['@id']
    child_res = testapp.post_json('/testing-link-sources/', {'target': parent_id})
    child_id = child_res.json['@graph'][0]['@id']
    return {'@id': parent_id, 'child': child_id}

def test_admin_post(testapp):
    testapp.post_json(COLLECTION_URL, item, status=201)
    testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=201)

def test_submitter_post(submitter_testapp):
    testapp = submitter_testapp
    testapp.post_json(COLLECTION_URL, item, status=201)
    res = testapp.post_json(COLLECTION_URL, item_with_uuid[0], status=422)
    assert any(error.get('name') == ['uuid'] for error in res.json['errors'])

def test_admin_put_uuid(content, testapp):
    url = content['@id']
    # so long as the same uuid is supplied, PUTing the uuid is fine
    testapp.put_json(url, item_with_uuid[0], status=200)
    # but the uuid may not be changed on PUT;
    testapp.put_json(url, item_with_uuid[1], status=422)

def test_submitter_put_uuid(content, submitter_testapp):
    testapp = submitter_testapp
    url = content['@id']
    # so long as the same uuid is supplied, PUTing the uuid is fine
    testapp.put_json(url, item_with_uuid[0], status=200)
    # but the uuid may not be changed on PUT;
    testapp.put_json(url, item_with_uuid[1], status=422)

def test_defaults_on_put(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'

    res = testapp.put_json(url, simple1, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.put_json(url, simple2, status=200)
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'
    assert res.json['@graph'][0]['simple2'] == 'supplied simple2'

def test_patch(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {'simple1': 'supplied simple1'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'simple2 default'

    res = testapp.patch_json(url, {'simple2': 'supplied simple2'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'supplied simple1'
    assert res.json['@graph'][0]['simple2'] == 'supplied simple2'

def test_patch_new_schema_version(content, root, testapp, monkeypatch):
    collection = root['testing_post_put_patch']
    properties = collection.type_info.schema['properties']

    url = content['@id']
    res = testapp.get(url)
    assert res.json['schema_version'] == '1'

    monkeypatch.setitem(properties['schema_version'], 'default', '2')
    monkeypatch.setattr(collection.type_info, 'schema_version', '2')
    monkeypatch.setitem(properties, 'new_property', {'default': 'new'})
    res = testapp.patch_json(url, {}, status=200)
    assert res.json['@graph'][0]['schema_version'] == '2'
    assert res.json['@graph'][0]['new_property'] == 'new'

def test_admin_put_protected_link(link_targets, testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location

    testapp.put_json(url, item_with_link[0], status=200)
    testapp.put_json(url, item_with_link[1], status=200)

def test_submitter_put_protected_link(link_targets, testapp, submitter_testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location

    submitter_testapp.put_json(url, item_with_link[0], status=200)
    submitter_testapp.put_json(url, item_with_link[1], status=422)

def test_put_object_not_touching_children(content_with_child, testapp):
    """
    The rev_link style of editing children is removed as of FF-1089
    This will still pass
    """
    url = content_with_child['@id']
    res = testapp.put_json(url, {}, status=200)
    assert content_with_child['child'] in res.json['@graph'][0]['reverse']

def test_put_object_editing_child_does_not_work(content_with_child, testapp):
    edit = {
        'reverse': [{
            '@id': content_with_child['child'],
            'status': 'released',
        }]
    }
    testapp.put_json(content_with_child['@id'], edit, status=200)
    res = testapp.get(content_with_child['child'] + '?frame=embedded')
    assert 'status' not in res.json

def test_post_object_with_child(content_with_child, testapp):
    """
    This will not post a rev link, since editing children through reverse links
    is no longer done
    """
    edit = {
        'reverse': [
            {'status': 'released'}
        ]
    }
    res = testapp.post_json('/testing-link-targets', edit, status=201)
    assert len(res.json['@graph'][0]['reverse']) == 0

def test_retry(testapp):
    res = testapp.post_json('/testing-post-put-patch/', {'required': ''})
    url = res.location
    res = testapp.get(url + '/@@testing-retry?datstore=database')
    assert res.json['attempt'] == 2
    assert not res.json['detached']

def test_post_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    #if we post this data it will fail with uuid conflict, as calling the human fixture posts it
    testapp.post_json('/organism/', human_data, status=409)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    rest = testapp.post_json('/organism/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_put_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    #if we post this data it will fail with invalid status

    testapp.post_json('/organism/', human_data, status=409)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    rest = testapp.post_json('/organism/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_post_check_only_invalid_data(testapp, human_data):
    '''
    note theese test should work on any object
    '''
    human_data['taxon_id'] = 24;
    testapp.post_json('/organism/?check_only=True', human_data, status=422)

def test_put_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    # human_data has already been posted, now put with invalid status
    human_data['status'] = 'no a valid status'
    testapp.put_json('/organisms/human/?check_only=True', human_data, status=422)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    human_data['status'] = human['status']
    rest = testapp.put_json('/organisms/human/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_patch_check_only(testapp, human_data, human):
    '''
    organism should validate fine but not post
    '''
    # human_data has already been posted, now put with invalid status
    human_data['status'] = 'no a valid status'
    testapp.patch_json('/organisms/human/?check_only=True', human_data, status=422)

    # so this one won't post, but schema validation is ok,
    # note it doesn't detect primary key
    human_data['status'] = human['status']
    rest = testapp.patch_json('/organisms/human/?check_only=True', human_data).json
    assert rest['status'] == 'success'

def test_patch_delete_fields(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'

    res = testapp.patch_json(url, {'simple1': 'this is a test'}, status=200)
    assert res.json['@graph'][0]['simple1'] == 'this is a test'

    # delete fields with defaults resets to default, while deleting non default field
    # completely removes them
    res = testapp.patch_json(url + "?delete_fields=simple1,field_no_default", {}, status=200)
    assert 'field_no_default' not in res.json['@graph'][0].keys()
    assert res.json['@graph'][0]['simple1'] == 'simple1 default'

def test_patch_delete_fields_non_string(content, testapp):
    url = content['@id']
    res = testapp.get(url)

    # delete fields with defaults resets to default, while deleting non default field
    # completely removes them
    res = testapp.patch_json(url + "?delete_fields=schema_version", {}, status=200)
    assert res.json['@graph'][0]['schema_version'] == '1'

def test_patch_delete_fields_still_works_with_no_validation(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'

    # with validate=false, then defaults are not populated so default fields are also deleted
    res = testapp.patch_json(url + "?delete_fields=simple1,field_no_default&validate=false", {}, status=200)
    assert 'field_no_default' not in res.json['@graph'][0].keys()
    assert 'simple1' not in res.json['@graph'][0].keys()

def test_patch_delete_fields_bad_param(content, testapp):
    url = content['@id']
    res = testapp.get(url)
    assert res.json['simple1'] == 'simple1 default'
    assert res.json['simple2'] == 'simple2 default'
    assert res.json['field_no_default'] == 'test'
    res = testapp.patch_json(url + "?delete_fields=simple1,bad_fieldname", {}, status=422)
    assert res.json['description'] == "Failed validation"
    assert res.json['errors'][0]['description'] == "Additional properties are not allowed ('bad_fieldname' was unexpected)"

def test_patch_delete_fields_import_items_admin(link_targets, testapp):
    res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201)
    url = res.location
    assert res.json['@graph'][0]['protected_link']
    res = testapp.patch_json(url + "?delete_fields=protected_link", {}, status=200)

def test_patch_delete_fields_import_items_submitter(content, submitter_testapp):
    testapp = submitter_testapp
    url = content['@id']
    res = testapp.get(url)
    assert res.json['protected']
    res = testapp.patch_json(url + "?delete_fields=protected", {}, status=422)

def test_patch_delete_fields_required(content, testapp):
    url = content['@id']
    res = testapp.get(url)

    # with validate=false, then defaults are not populated so default fields are also deleted
    res = testapp.patch_json(url + "?delete_fields=required", {}, status=422)
    assert res.json['description'] == "Failed validation"
    assert res.json['errors'][0]['description'] == "'required' is a required property"
<|endoftext|>


<commit_before>''' Provide a functions and classes to implement a custom JSON encoder for
serializing objects for BokehJS.

The primary interface is provided by the |serialize_json| function, which
uses the custom |BokehJSONEncoder| to produce JSON output.

In general, functions in this module convert values in the following way:

* Datetime values (Python, Pandas, NumPy) are converted to floating point
  milliseconds since epoch.

* TimeDelta values are converted to absolute floating point milliseconds.

* RelativeDelta values are converted to dictionaries.

* Decimal values are converted to floating point.

* Sequences (Pandas Series, NumPy arrays, python sequences) that are passed
  though this interface are converted to lists. Note, however, that arrays in
  data sources inside Bokeh Documents are converted elsewhere, and by default
  use a binary encoded format.

* Bokeh ``Model`` instances are usually serialized elsewhere in the context
  of an entire Bokeh Document. Models passed trough this interface are
  converted to references.

* ``HasProps`` (that are not Bokeh models) are converted to key/value dicts or
  all their properties and values.

* ``Color`` instances are converted to CSS color values.

.. |serialize_json| replace:: :class:`~bokeh.core.json_encoder.serialize_json`
.. |BokehJSONEncoder| replace:: :class:`~bokeh.core.json_encoder.BokehJSONEncoder`

'''
from __future__ import absolute_import

import logging
log = logging.getLogger(__name__)

import collections
import decimal
import json

import numpy as np

from ..settings import settings
from ..util.dependencies import import_optional
from ..util.serialization import convert_datetime_type, is_datetime_type, transform_series, transform_array

pd = import_optional('pandas')
rd = import_optional("dateutil.relativedelta")

class BokehJSONEncoder(json.JSONEncoder):
    ''' A custom ``json.JSONEncoder`` subclass for encoding objects in
    accordance with the BokehJS protocol.

    '''
    def transform_python_types(self, obj):
        ''' Handle special scalars such as (Python, NumPy, or Pandas)
        datetimes, or Decimal values.

        Args:
            obj (obj) :

                The object to encode. Anything not specifically handled in
                this method is passed on to the default system JSON encoder.

        '''

        # date/time values that get serialized as milliseconds
        if is_datetime_type(obj):
            return convert_datetime_type(obj)

        # slice objects
        elif isinstance(obj, slice):
            return dict(start=obj.start, stop=obj.stop, step=obj.step)

        # NumPy scalars
        elif np.issubdtype(type(obj), np.float):
            return float(obj)
        elif np.issubdtype(type(obj), np.integer):
            return int(obj)
        elif np.issubdtype(type(obj), np.bool_):
            return bool(obj)

        # Decimal values
        elif isinstance(obj, decimal.Decimal):
            return float(obj)

        # RelativeDelta gets serialized as a dict
        elif rd and isinstance(obj, rd.relativedelta):
            return dict(years=obj.years,
                    months=obj.months,
                    days=obj.days,
                    hours=obj.hours,
                    minutes=obj.minutes,
                    seconds=obj.seconds,
                    microseconds=obj.microseconds)

        else:
            return super(BokehJSONEncoder, self).default(obj)

    def default(self, obj):
        ''' The required ``default`` method for JSONEncoder subclasses.

        Args:
            obj (obj) :

                The object to encode. Anything not specifically handled in
                this method is passed on to the default system JSON encoder.

        '''

        from ..model import Model
        from ..colors import Color
        from .has_props import HasProps

        # array types -- use force_list here, only binary
        # encoding CDS columns for now
        if pd and isinstance(obj, (pd.Series, pd.Index)):
            return transform_series(obj, force_list=True)
        elif isinstance(obj, np.ndarray):
            return transform_array(obj, force_list=True)
        elif isinstance(obj, collections.deque):
            return list(map(self.default, obj))
        elif isinstance(obj, Model):
            return obj.ref
        elif isinstance(obj, HasProps):
            return obj.properties_with_values(include_defaults=False)
        elif isinstance(obj, Color):
            return obj.to_css()

        else:
            return self.transform_python_types(obj)

def serialize_json(obj, pretty=None, indent=None, **kwargs):
    ''' Return a serialized JSON representation of objects, suitable to
    send to BokehJS.

    This function is typically used to serialize single python objects in
    the manner expected by BokehJS. In particular, many datetime values are
    automatically normalized to an expected format. Some Bokeh objects can
    also be passed, but note that Bokeh models are typically properly
    serialized in the context of an entire Bokeh document.

    The resulting JSON always has sorted keys. By default. the output is
    as compact as possible unless pretty output or indentation is requested.

    Args:
        obj (obj) : the object to serialize to JSON format

        pretty (bool, optional) :

            Whether to generate prettified output. If ``True``, spaces are
            added after added after separators, and indentation and newlines
            are applied. (default: False)

            Pretty output can also be enabled with the environment variable
            ``BOKEH_PRETTY``, which overrides this argument, if set.

        indent (int or None, optional) :

            Amount of indentation to use in generated JSON output. If ``None``
            then no indentation is used, unless pretty output is enabled,
            in which case two spaces are used. (default: None)

    Any additional keyword arguments are passed to ``json.dumps``, except for
    some that  are computed internally, and cannot be overridden:

    * allow_nan
    * indent
    * separators
    * sort_keys

    Examples:

        .. code-block:: python

            >>> data = dict(b=np.datetime64('2017-01-01'), a = np.arange(3))

            >>>print(serialize_json(data))
            {"a":[0,1,2],"b":1483228800000.0}

            >>> print(serialize_json(data, pretty=True))
            {
              "a": [
                0,
                1,
                2
              ],
              "b": 1483228800000.0
            }

    '''

    # these args to json.dumps are computed internally and should not be passed along
    for name in ['allow_nan', 'separators', 'sort_keys']:
        if name in kwargs:
            raise ValueError("The value of %r is computed internally, overriding is not permissable." % name)

    if pretty is None:
        pretty = settings.pretty(False)

    if pretty:
        separators=(",", ": ")
    else:
        separators=(",", ":")

    if pretty and indent is None:
        indent = 2

    return json.dumps(obj, cls=BokehJSONEncoder, allow_nan=False, indent=indent, separators=separators, sort_keys=True, **kwargs)
<commit_msg>Fixed FutureWarning in json serialization of floating values (#7533)<commit_after>''' Provide a functions and classes to implement a custom JSON encoder for
serializing objects for BokehJS.

The primary interface is provided by the |serialize_json| function, which
uses the custom |BokehJSONEncoder| to produce JSON output.

In general, functions in this module convert values in the following way:

* Datetime values (Python, Pandas, NumPy) are converted to floating point
  milliseconds since epoch.

* TimeDelta values are converted to absolute floating point milliseconds.

* RelativeDelta values are converted to dictionaries.

* Decimal values are converted to floating point.

* Sequences (Pandas Series, NumPy arrays, python sequences) that are passed
  though this interface are converted to lists. Note, however, that arrays in
  data sources inside Bokeh Documents are converted elsewhere, and by default
  use a binary encoded format.

* Bokeh ``Model`` instances are usually serialized elsewhere in the context
  of an entire Bokeh Document. Models passed trough this interface are
  converted to references.

* ``HasProps`` (that are not Bokeh models) are converted to key/value dicts or
  all their properties and values.

* ``Color`` instances are converted to CSS color values.

.. |serialize_json| replace:: :class:`~bokeh.core.json_encoder.serialize_json`
.. |BokehJSONEncoder| replace:: :class:`~bokeh.core.json_encoder.BokehJSONEncoder`

'''
from __future__ import absolute_import

import logging
log = logging.getLogger(__name__)

import collections
import decimal
import json

import numpy as np

from ..settings import settings
from ..util.dependencies import import_optional
from ..util.serialization import convert_datetime_type, is_datetime_type, transform_series, transform_array

pd = import_optional('pandas')
rd = import_optional("dateutil.relativedelta")

class BokehJSONEncoder(json.JSONEncoder):
    ''' A custom ``json.JSONEncoder`` subclass for encoding objects in
    accordance with the BokehJS protocol.

    '''
    def transform_python_types(self, obj):
        ''' Handle special scalars such as (Python, NumPy, or Pandas)
        datetimes, or Decimal values.

        Args:
            obj (obj) :

                The object to encode. Anything not specifically handled in
                this method is passed on to the default system JSON encoder.

        '''

        # date/time values that get serialized as milliseconds
        if is_datetime_type(obj):
            return convert_datetime_type(obj)

        # slice objects
        elif isinstance(obj, slice):
            return dict(start=obj.start, stop=obj.stop, step=obj.step)

        # NumPy scalars
        elif np.issubdtype(type(obj), np.floating):
            return float(obj)
        elif np.issubdtype(type(obj), np.integer):
            return int(obj)
        elif np.issubdtype(type(obj), np.bool_):
            return bool(obj)

        # Decimal values
        elif isinstance(obj, decimal.Decimal):
            return float(obj)

        # RelativeDelta gets serialized as a dict
        elif rd and isinstance(obj, rd.relativedelta):
            return dict(years=obj.years,
                    months=obj.months,
                    days=obj.days,
                    hours=obj.hours,
                    minutes=obj.minutes,
                    seconds=obj.seconds,
                    microseconds=obj.microseconds)

        else:
            return super(BokehJSONEncoder, self).default(obj)

    def default(self, obj):
        ''' The required ``default`` method for JSONEncoder subclasses.

        Args:
            obj (obj) :

                The object to encode. Anything not specifically handled in
                this method is passed on to the default system JSON encoder.

        '''

        from ..model import Model
        from ..colors import Color
        from .has_props import HasProps

        # array types -- use force_list here, only binary
        # encoding CDS columns for now
        if pd and isinstance(obj, (pd.Series, pd.Index)):
            return transform_series(obj, force_list=True)
        elif isinstance(obj, np.ndarray):
            return transform_array(obj, force_list=True)
        elif isinstance(obj, collections.deque):
            return list(map(self.default, obj))
        elif isinstance(obj, Model):
            return obj.ref
        elif isinstance(obj, HasProps):
            return obj.properties_with_values(include_defaults=False)
        elif isinstance(obj, Color):
            return obj.to_css()

        else:
            return self.transform_python_types(obj)

def serialize_json(obj, pretty=None, indent=None, **kwargs):
    ''' Return a serialized JSON representation of objects, suitable to
    send to BokehJS.

    This function is typically used to serialize single python objects in
    the manner expected by BokehJS. In particular, many datetime values are
    automatically normalized to an expected format. Some Bokeh objects can
    also be passed, but note that Bokeh models are typically properly
    serialized in the context of an entire Bokeh document.

    The resulting JSON always has sorted keys. By default. the output is
    as compact as possible unless pretty output or indentation is requested.

    Args:
        obj (obj) : the object to serialize to JSON format

        pretty (bool, optional) :

            Whether to generate prettified output. If ``True``, spaces are
            added after added after separators, and indentation and newlines
            are applied. (default: False)

            Pretty output can also be enabled with the environment variable
            ``BOKEH_PRETTY``, which overrides this argument, if set.

        indent (int or None, optional) :

            Amount of indentation to use in generated JSON output. If ``None``
            then no indentation is used, unless pretty output is enabled,
            in which case two spaces are used. (default: None)

    Any additional keyword arguments are passed to ``json.dumps``, except for
    some that  are computed internally, and cannot be overridden:

    * allow_nan
    * indent
    * separators
    * sort_keys

    Examples:

        .. code-block:: python

            >>> data = dict(b=np.datetime64('2017-01-01'), a = np.arange(3))

            >>>print(serialize_json(data))
            {"a":[0,1,2],"b":1483228800000.0}

            >>> print(serialize_json(data, pretty=True))
            {
              "a": [
                0,
                1,
                2
              ],
              "b": 1483228800000.0
            }

    '''

    # these args to json.dumps are computed internally and should not be passed along
    for name in ['allow_nan', 'separators', 'sort_keys']:
        if name in kwargs:
            raise ValueError("The value of %r is computed internally, overriding is not permissable." % name)

    if pretty is None:
        pretty = settings.pretty(False)

    if pretty:
        separators=(",", ": ")
    else:
        separators=(",", ":")

    if pretty and indent is None:
        indent = 2

    return json.dumps(obj, cls=BokehJSONEncoder, allow_nan=False, indent=indent, separators=separators, sort_keys=True, **kwargs)
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

# usage: python exp11a.py bigk.r.i.t 0 0
# system arg

artist = ''
SMALL_FLAG = 1
FAST_FLAG = 1
try:
	artist = sys.argv[1]
	SMALL_FLAG = int(sys.argv[2])
	FAST_FLAG = int(sys.argv[3])
except Exception, e:
	print('-- {}'.format(e))

print('-- Artist: {}'.format(artist))
print('-- SMALL FLAG: {}'.format(SMALL_FLAG))
print('-- FAST FLAG: {}'.format(FAST_FLAG))

# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def conv2d(x, W):
	return tf.nn.conv2d(x, W, [1, 1, 1, 1], 'VALID')

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

def max_pool(x, p):
  return tf.nn.max_pool(x, ksize=[1, p, p, 1],
                        strides=[1, p, p, 1], padding='VALID')

def batch_eval(data, label, metric, batch_size=256):
	value = 0.
	for i in range(0, data.shape[0], batch_size):
		batch_end_point = min(i + batch_size, data.shape[0])
		batch_data = data[i : batch_end_point]
		batch_label = label[i : batch_end_point]
		value += batch_data.shape[0] * metric.eval(feed_dict={x: batch_data, y_: batch_label, keep_prob: 1.0})
	value = value / data.shape[0]
	return value

def MRR_batch(data, label, batch_size=256):
	value = 0.
	for i in range(0, data.shape[0], batch_size):
		batch_end_point = min(i + batch_size, data.shape[0])
		batch_data = data[i : batch_end_point]
		batch_label = label[i : batch_end_point]
		batch_pred = sess.run(y_sm, feed_dict={x: batch_data, y_: batch_label, keep_prob: 1.0})
		value += batch_data.shape[0] * MRR(batch_pred, batch_label)
	value = value / data.shape[0]
	return value

def MRR(pred, label):
	'''
		pred, label are np arrays with dimension m x k
	'''
	pred_rank = np.argsort(-pred, axis=1)
	groundtruth = np.argmax(label, axis=1)
	rank = np.array([np.where(pred_rank[index] == groundtruth[index])[0].item(0) + 1 \
		for index in range(label.shape[0])])
	return np.mean(1. / rank)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 11a: MNIST Mirror on Full Window...')
sys_path = '/pylon2/ci560sp/haunter/'
filename = artist + '_data.mat'
if SMALL_FLAG:
	filename = artist + '_data_small.mat'
filepath = sys_path + filename
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
D_train = np.array(f.get('DTrain'))
D_val = np.array(f.get('DVal'))
np.random.shuffle(D_train)
np.random.shuffle(D_val)
X_train = D_train[:, :-1]
y_train = D_train[:, -1]
X_val = D_val[:, :-1]
y_val = D_val[:, -1]

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del D_train, D_val, f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_train, total_features = X_train.shape
num_freq = 121
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

# ==============================================
# First layer
# ==============================================
r1, c1, k1 = 5, 5, 32
W_conv1 = init_weight_variable([r1, c1, 1, k1])
b_conv1 = init_bias_variable([k1])
x_image = tf.reshape(x, [-1, num_freq, num_frames, 1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool(h_conv1, 2)
h1r, h1c = (num_freq - r1 + 1) / 2, (num_frames - c1 + 1) / 2
# ==============================================
# Second layer
# ==============================================
r2, c2, k2 = 5, 5, 64
W_conv2 = init_weight_variable([r2, c2, k1, k2])
b_conv2 = init_bias_variable([k2])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool(h_conv2, 2)
h2r, h2c = (h1r - r2 + 1) / 2, (h1c - c2 + 1) / 2
h_pool2_flat = tf.reshape(h_pool2, [-1, h2r * h2c * k2])

# ==============================================
# Dense layer
# ==============================================
nhidden = 1024
W_fc1 = init_weight_variable([h2r * h2c * k2, nhidden])
b_fc1 = init_bias_variable([nhidden])
h_fc1 = tf.nn.relu(batch_nm(tf.matmul(h_pool2_flat, W_fc1) + b_fc1))
# dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

W_sm = init_weight_variable([nhidden, num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(h_fc1_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=2.5e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)
y_val = sess.run(y_val_OHEnc)

# evaluation metrics
train_err_list = []
val_err_list = []
val_mrr_list = []

# saver setup
varsave_list = [W_conv1, b_conv1, W_conv2, b_conv2, W_fc1, b_fc1, W_sm, b_sm]
saver = tf.train.Saver(varsave_list)
save_path = './out/11amodel_{}.ckpt'.format(artist)
opt_val_err = np.inf
opt_epoch = -1
step_counter = 0
max_counter = 5000

batch_size = 256
max_epochs = 500
print_freq = 200
num_iter = 0

if FAST_FLAG:
	max_epochs = 1
	print_freq = 10
print('==> Training the full network...')
t_start = time.time()
for epoch in range(max_epochs):
	for i in range(0, num_train, batch_size):
		batch_end_point = min(i + batch_size, num_train)
		train_batch_data = X_train[i : batch_end_point]
		train_batch_label = y_train[i : batch_end_point]
		train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
		if (num_iter + 1) % print_freq == 0:
			# evaluate metrics
			train_err = cross_entropy.eval(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 1.0})
			train_err_list.append(train_err)
			val_err = batch_eval(X_val, y_val, cross_entropy)
			val_err_list.append(val_err)
			val_mrr = MRR_batch(X_val, y_val)
			val_mrr_list.append(val_mrr)
			print("-- epoch %d, iter %d, training error %g, validation error %g"%(epoch + 1, num_iter + 1, train_err, val_err))
			# save screenshot of the model
			if val_err < opt_val_err:
				step_counter = 0
				saver.save(sess, save_path)
				print('==> New optimal validation error found. Model saved.')
				opt_val_err, opt_epoch, opt_iter = val_err, epoch + 1, num_iter + 1
		if step_counter > max_counter:
			print('==> Step counter exceeds maximum value. Stop training at epoch {}, iter {}.'.format(epoch + 1, num_iter + 1))
			break
		step_counter += 1
		num_iter += 1
t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Restore model & Evaluations
# ==============================================
saver.restore(sess, save_path)
print('==> Model restored to epoch {}, iter {}'.format(opt_epoch, opt_iter))

from scipy.io import savemat
model_path = './out/11a_{}'.format(artist)
W1, W2 = sess.run([W_conv1, W_conv2], feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
savemat(model_path, {'W1': W1, 'W2': W2})
print('==> CNN filters saved to {}.mat'.format(model_path))

print('-- Final Validation error: {:.4E}'.format(batch_eval(X_val, y_val, cross_entropy)))
print('-- Final Validation MRR: {:.3f}'.format(MRR_batch(X_val, y_val)))

print('-- Training error --')
print([float('{:.4E}'.format(e)) for e in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(e)) for e in val_err_list])
print('-- Validaiton MRR --')
print([float('{:.3f}'.format(e)) for e in val_mrr_list])

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_err_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b', label='training')
val_err_plot = plt.plot(x_list, val_err_list , color='orange', label='validation')
plt.xlabel('Number of Iterations')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs for {}'.format(artist))
plt.legend(loc='best')
plt.savefig('./out/exp11a_{}.png'.format(artist), format='png')
plt.close()

print('==> Finished!')
<commit_msg>fixing unbreak<commit_after>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

# usage: python exp11a.py bigk.r.i.t 0 0
# system arg

artist = ''
SMALL_FLAG = 1
FAST_FLAG = 1
try:
	artist = sys.argv[1]
	SMALL_FLAG = int(sys.argv[2])
	FAST_FLAG = int(sys.argv[3])
except Exception, e:
	print('-- {}'.format(e))

print('-- Artist: {}'.format(artist))
print('-- SMALL FLAG: {}'.format(SMALL_FLAG))
print('-- FAST FLAG: {}'.format(FAST_FLAG))

# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def conv2d(x, W):
	return tf.nn.conv2d(x, W, [1, 1, 1, 1], 'VALID')

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

def max_pool(x, p):
  return tf.nn.max_pool(x, ksize=[1, p, p, 1],
                        strides=[1, p, p, 1], padding='VALID')

def batch_eval(data, label, metric, batch_size=256):
	value = 0.
	for i in range(0, data.shape[0], batch_size):
		batch_end_point = min(i + batch_size, data.shape[0])
		batch_data = data[i : batch_end_point]
		batch_label = label[i : batch_end_point]
		value += batch_data.shape[0] * metric.eval(feed_dict={x: batch_data, y_: batch_label, keep_prob: 1.0})
	value = value / data.shape[0]
	return value

def MRR_batch(data, label, batch_size=256):
	value = 0.
	for i in range(0, data.shape[0], batch_size):
		batch_end_point = min(i + batch_size, data.shape[0])
		batch_data = data[i : batch_end_point]
		batch_label = label[i : batch_end_point]
		batch_pred = sess.run(y_sm, feed_dict={x: batch_data, y_: batch_label, keep_prob: 1.0})
		value += batch_data.shape[0] * MRR(batch_pred, batch_label)
	value = value / data.shape[0]
	return value

def MRR(pred, label):
	'''
		pred, label are np arrays with dimension m x k
	'''
	pred_rank = np.argsort(-pred, axis=1)
	groundtruth = np.argmax(label, axis=1)
	rank = np.array([np.where(pred_rank[index] == groundtruth[index])[0].item(0) + 1 \
		for index in range(label.shape[0])])
	return np.mean(1. / rank)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 11a: MNIST Mirror on Full Window...')
sys_path = '/pylon2/ci560sp/haunter/'
filename = artist + '_data.mat'
if SMALL_FLAG:
	filename = artist + '_data_small.mat'
filepath = sys_path + filename
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
D_train = np.array(f.get('DTrain'))
D_val = np.array(f.get('DVal'))
np.random.shuffle(D_train)
np.random.shuffle(D_val)
X_train = D_train[:, :-1]
y_train = D_train[:, -1]
X_val = D_val[:, :-1]
y_val = D_val[:, -1]

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del D_train, D_val, f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_train, total_features = X_train.shape
num_freq = 121
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

# ==============================================
# First layer
# ==============================================
r1, c1, k1 = 5, 5, 32
W_conv1 = init_weight_variable([r1, c1, 1, k1])
b_conv1 = init_bias_variable([k1])
x_image = tf.reshape(x, [-1, num_freq, num_frames, 1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool(h_conv1, 2)
h1r, h1c = (num_freq - r1 + 1) / 2, (num_frames - c1 + 1) / 2
# ==============================================
# Second layer
# ==============================================
r2, c2, k2 = 5, 5, 64
W_conv2 = init_weight_variable([r2, c2, k1, k2])
b_conv2 = init_bias_variable([k2])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool(h_conv2, 2)
h2r, h2c = (h1r - r2 + 1) / 2, (h1c - c2 + 1) / 2
h_pool2_flat = tf.reshape(h_pool2, [-1, h2r * h2c * k2])

# ==============================================
# Dense layer
# ==============================================
nhidden = 1024
W_fc1 = init_weight_variable([h2r * h2c * k2, nhidden])
b_fc1 = init_bias_variable([nhidden])
h_fc1 = tf.nn.relu(batch_nm(tf.matmul(h_pool2_flat, W_fc1) + b_fc1))
# dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

W_sm = init_weight_variable([nhidden, num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(h_fc1_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=2.5e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)
y_val = sess.run(y_val_OHEnc)

# evaluation metrics
train_err_list = []
val_err_list = []
val_mrr_list = []

# saver setup
varsave_list = [W_conv1, b_conv1, W_conv2, b_conv2, W_fc1, b_fc1, W_sm, b_sm]
saver = tf.train.Saver(varsave_list)
save_path = './out/11amodel_{}.ckpt'.format(artist)
opt_val_err = np.inf
opt_epoch = -1
step_counter = 0
max_counter = 5000

batch_size = 256
max_epochs = 500
print_freq = 200
num_iter = 0

if FAST_FLAG:
	max_epochs = 1
	print_freq = 10
print('==> Training the full network...')
t_start = time.time()
for epoch in range(max_epochs):
	if step_counter <= max_counter:
		for i in range(0, num_train, batch_size):
			batch_end_point = min(i + batch_size, num_train)
			train_batch_data = X_train[i : batch_end_point]
			train_batch_label = y_train[i : batch_end_point]
			train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
			if (num_iter + 1) % print_freq == 0:
				# evaluate metrics
				train_err = cross_entropy.eval(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 1.0})
				train_err_list.append(train_err)
				val_err = batch_eval(X_val, y_val, cross_entropy)
				val_err_list.append(val_err)
				val_mrr = MRR_batch(X_val, y_val)
				val_mrr_list.append(val_mrr)
				print("-- epoch %d, iter %d, training error %g, validation error %g"%(epoch + 1, num_iter + 1, train_err, val_err))
				# save screenshot of the model
				if val_err < opt_val_err:
					step_counter = 0
					saver.save(sess, save_path)
					print('==> New optimal validation error found. Model saved.')
					opt_val_err, opt_epoch, opt_iter = val_err, epoch + 1, num_iter + 1
			if step_counter > max_counter:
				print('==> Step counter exceeds maximum value. Stop training at epoch {}, iter {}.'.format(epoch + 1, num_iter + 1))
				break
			step_counter += 1
			num_iter += 1
	else:
		break

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Restore model & Evaluations
# ==============================================
saver.restore(sess, save_path)
print('==> Model restored to epoch {}, iter {}'.format(opt_epoch, opt_iter))

from scipy.io import savemat
model_path = './out/11a_{}'.format(artist)
W1, W2 = sess.run([W_conv1, W_conv2], feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
savemat(model_path, {'W1': W1, 'W2': W2})
print('==> CNN filters saved to {}.mat'.format(model_path))

print('-- Final Validation error: {:.4E}'.format(batch_eval(X_val, y_val, cross_entropy)))
print('-- Final Validation MRR: {:.3f}'.format(MRR_batch(X_val, y_val)))

print('-- Training error --')
print([float('{:.4E}'.format(e)) for e in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(e)) for e in val_err_list])
print('-- Validaiton MRR --')
print([float('{:.3f}'.format(e)) for e in val_mrr_list])

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_err_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b', label='training')
val_err_plot = plt.plot(x_list, val_err_list , color='orange', label='validation')
plt.xlabel('Number of Iterations')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs for {}'.format(artist))
plt.legend(loc='best')
plt.savefig('./out/exp11a_{}.png'.format(artist), format='png')
plt.close()

print('==> Finished!')
<|endoftext|>


<commit_before>
from astropy.io import fits
import numpy as np
from astropy import log
from astropy.utils.console import ProgressBar
import os

def create_huge_fits(filename, header, shape=None, verbose=True,
                     return_hdu=False, dtype=None, fill_nan=False):
    '''
    Create empty FITS files too large to fit into memory.
    '''

    if os.path.exists(filename):
        raise IOError("{} already exists.".format(filename))

    # Add shape info to the header
    if "NAXIS" not in header and shape is None:
        raise TypeError("shape must be given when the header does not have "
                        "shape information ('NAXIS').")

    if shape is None:
        shape = (header["NAXIS3"], header["NAXIS2"], header["NAXIS1"])

    if dtype is None:
        dtype = fits.BITPIX2DTYPE[header['BITPIX']]
    else:
        # Make sure the header has the right BITPIX
        header['BITPIX'] = fits.DTYPE2BITPIX[dtype.name]

    # Iterate over the smallest axis
    min_axis = np.array(shape).argmin()
    plane_shape = [sh for i, sh in enumerate(shape) if i != min_axis]
    fill_plane = np.zeros(plane_shape, dtype=dtype)
    if fill_nan:
        fill_plane *= np.NaN

    if verbose:
        iterat = ProgressBar(shape[min_axis])
    else:
        iterat = xrange(shape[min_axis])

    output_fits = fits.StreamingHDU(filename, header)

    for chan in iterat:
        output_fits.write(fill_plane)

    output_fits.close()

    if return_hdu:
        output_fits = fits.open(filename, mode='update')
        return output_fits

def save_to_huge_fits(filename, cube, verbose=True, overwrite=False,
                      chunk=100):
    '''
    Save a huge SpectralCube by streaming the cube per channel.
    '''

    if os.path.exists(filename):
        if overwrite:
            raise IOError("{} already exists. Delete the file or enable "
                          "'overwrite'.".format(filename))
        output_fits = fits.open(filename, mode='update')
    else:
        output_fits = create_huge_fits(filename, cube.header, verbose=verbose,
                                       dtype=cube[:, 0, 0].dtype,
                                       return_hdu=True)

    for chan in xrange(cube.shape[0]):
        plane = cube[chan]
        if hasattr(plane, 'unit'):
            plane = plane.value

        output_fits[0][chan, :, :] = plane

        if chan % chunk == 0:
            output_fits.flush()

    output_fits.flush()
<commit_msg>Deal with the case where a string has been passed as the dtype<commit_after>
from astropy.io import fits
import numpy as np
from astropy import log
from astropy.utils.console import ProgressBar
import os

def create_huge_fits(filename, header, shape=None, verbose=True,
                     return_hdu=False, dtype=None, fill_nan=False):
    '''
    Create empty FITS files too large to fit into memory.
    '''

    if os.path.exists(filename):
        raise IOError("{} already exists.".format(filename))

    # Add shape info to the header
    if "NAXIS" not in header and shape is None:
        raise TypeError("shape must be given when the header does not have "
                        "shape information ('NAXIS').")

    if shape is None:
        shape = (header["NAXIS3"], header["NAXIS2"], header["NAXIS1"])

    if dtype is None:
        dtype = fits.BITPIX2DTYPE[header['BITPIX']]
    else:
        # Make sure the header has the right BITPIX
        name = dtype.name if hasattr(dtype, "name") else dtype
        header['BITPIX'] = fits.DTYPE2BITPIX[name]

    # Iterate over the smallest axis
    min_axis = np.array(shape).argmin()
    plane_shape = [sh for i, sh in enumerate(shape) if i != min_axis]
    fill_plane = np.zeros(plane_shape, dtype=dtype)
    if fill_nan:
        fill_plane *= np.NaN

    if verbose:
        iterat = ProgressBar(shape[min_axis])
    else:
        iterat = xrange(shape[min_axis])

    output_fits = fits.StreamingHDU(filename, header)

    for chan in iterat:
        output_fits.write(fill_plane)

    output_fits.close()

    if return_hdu:
        output_fits = fits.open(filename, mode='update')
        return output_fits

def save_to_huge_fits(filename, cube, verbose=True, overwrite=False,
                      chunk=100):
    '''
    Save a huge SpectralCube by streaming the cube per channel.
    '''

    if os.path.exists(filename):
        if overwrite:
            raise IOError("{} already exists. Delete the file or enable "
                          "'overwrite'.".format(filename))
        output_fits = fits.open(filename, mode='update')
    else:
        output_fits = create_huge_fits(filename, cube.header, verbose=verbose,
                                       dtype=cube[:, 0, 0].dtype,
                                       return_hdu=True)

    for chan in xrange(cube.shape[0]):
        plane = cube[chan]
        if hasattr(plane, 'unit'):
            plane = plane.value

        output_fits[0][chan, :, :] = plane

        if chan % chunk == 0:
            output_fits.flush()

    output_fits.flush()
<|endoftext|>