<commit_before>"""
Python parser for parsing code into a JSON object
Accepts input from the command-line either as STDIN
or a raw string.

Example usage:

  python src/parser.py "$(cat test/data/bye.py)"
  cat test/data/bye.py | python src/parser.py

Most credit goes to https://github.com/exana/python-ast-explorer
for parsing ideas
"""

import sys
import json
import ast
import io
import tokenize
import token
import collections

Token = collections.namedtuple("Token", "type string lineno col_offset start end")

from pprint import pprint
from argparse import ArgumentParser

TYPE = 0
STRING = 1
LINENO = 2
COL_OFFSET = 3
START = 4
END = 5

def fix_ast(tree, source_lines, tokens):
    def match_tokens(node):
        """
        Match tokens with this node
        """
        lineno, col_offset = node.lineno, node.col_offset

        # Make sure you filter out tokens we're not interested in, particularly
        # DEDENTs There are probably other types we'll come across that need
        # filtered out, so you can find the list here:
        # https://docs.python.org/2/library/token.html#token.ENDMARKER
        possible_tokens = list(filter((lambda tok: tok[LINENO] == lineno
                                       and tok[COL_OFFSET] == col_offset
                                       and tok[TYPE] not in [token.DEDENT]), tokens))

        # print possible_tokens
        if len(possible_tokens) > 0:
            tok = possible_tokens.pop(0)
            node.string = tok[STRING]
            node.start = tok[START]
            node.end = tok[END]

    for node in ast.walk(tree):
        if hasattr(node, 'lineno') and hasattr(node, 'col_offset'):
            match_tokens(node)

    def fix_ends(node, level=0):
        nodeEnd = node.end if hasattr(node, 'end') else None

        for field, value in ast.iter_fields(node):
             if isinstance(value, list):
                for item in value:
                    if isinstance(item, ast.AST):
                        fix_ends(item, level=level+1)
                        nodeEnd = max(nodeEnd, item.end) if hasattr(item, 'end') else nodeEnd
             elif isinstance(value, ast.AST):
                fix_ends(value, level=level+1)
             nodeEnd = max(nodeEnd, value.end) if hasattr(value, 'end') else nodeEnd

        node.end = nodeEnd

    fix_ends(tree)
    return tree

def tokenize_with_char_offsets(source):
    readline = io.StringIO(source).readline
    encoding = 'utf8'
    token_source = tokenize.generate_tokens(readline)
    char_lines = list(map(lambda line: line + "\n", source.split("\n")))
    line_lens = [ len(line) for line in char_lines ]

    # pprint(['char_lines', char_lines])
    # pprint(['line_lens', line_lens])

    byte_lines = None
    tokens = []

    for token in token_source:
        # pprint('\n')
        # pprint(['token', token])
        # if token[TYPE] == encoding:
            # first token
            # encoding = token[STRING]
        byte_lines = list(map(lambda line: line.encode(encoding), char_lines))

        if token[LINENO][0] == 0 or (token[LINENO][1] == 0 and token[COL_OFFSET][1] == 0):
            # q: what's even the point of these tokens?
            # a: at least DEDENT tokens. they confuse the future steps though, so filter them out downstream
            tok = Token(token[TYPE], token[STRING],
                        token[LINENO][0], token[LINENO][1],
                        token[COL_OFFSET][0], token[COL_OFFSET][1])

        else:
            assert token[LINENO][0] > 0 # lineno is > 0

            start_line = token[LINENO][0]  # closed [ e.g. includes
            start_char = token[LINENO][1]
            end_line = token[COL_OFFSET][0]
            end_char = token[COL_OFFSET][1] # open ) (e.g. minus 1 to include)

            byte_start_line = byte_lines[start_line-1]
            char_start_col = len(byte_start_line[:start_char].decode(encoding))

            byte_end_line = byte_lines[token[COL_OFFSET][0]-1]
            char_end_col = len(byte_end_line[:token[COL_OFFSET][1]].decode(encoding))

            # pprint(['start_line', start_line, range(1, start_line)])

            start_lines_offset = reduce(lambda x, y: x + line_lens[y-1], range(1, start_line), 0)
            end_lines_offset = reduce(lambda x, y: x + line_lens[y-1], range(1, end_line), 0)

            # start position = sum(prevlines) + starting_char
            # end position = sum(prevlines) +  sum(ending_char)

            # bug in start_lines_offset
            # pprint(['start_lines_offset', 'char_start_col', start_lines_offset, char_start_col])
            # pprint(['end_lines_offset', 'char_end_col', end_lines_offset, char_end_col])

            start_char_offset = start_lines_offset + char_start_col
            end_char_offset = end_lines_offset + char_end_col

            tok = Token(token[TYPE], token[STRING],
                        start_line, start_char,
                        start_char_offset, end_char_offset)

            # pprint(tok)

        # pprint(tok)
        tokens.append(tok)

    return tokens

def parse_ast(node, tokens, code=None):
    """
    Parse the ast into a reasonable JSON object
    """
    if isinstance(node, ast.Module):
        # Just starting out
        result = {
            "type": "Program",
            "start": 0,
            "end": len(code), # hmm
            "body": parse_ast(node.body, tokens)
        }
        return result
    # node or list?
    if isinstance(node, list):
        result = []
        for child_node in node:
            result += [parse_ast(child_node, tokens)]
        return result

    if isinstance(node, ast.AST):
        result = {"type": node.__class__.__name__}
        for k in node.__dict__:
            val = getattr(node, k)
            # if isinstance(val, ast.Str):
                # result["id"] = {"type": "Identifier", "name": node.value.s}
                # result[k] = parse_ast(val, tokens)
            # else:
            result[k] = parse_ast(val, tokens)

        return result

    return node

def make_ast(code, mode='exec'):
    """
    Make an AST tree from a string of code
    """
    code = unicode(code)
    tree = ast.parse(code, '<unknown>', mode)
    tokens = tokenize_with_char_offsets(code)
    tree = fix_ast(tree, code, tokens)
    return parse_ast(tree, tokens, code)

def read_from_stdin():
    """
    Read from stdin helper
    """
    while True:
        output = sys.stdin.read()
        if not output:
            print '[No more data]'
            break
        return output

def main():
    """
    Main
    """
    cli_parser = ArgumentParser(
        description="Convert python code into a JSON tree")
    cli_parser.add_argument(
        'text', nargs='?', default=None, action='store', help='Code to parse')
    args = cli_parser.parse_args()

    code = args.text if args.text is not None else read_from_stdin()
    tree = make_ast(code)

    print json.dumps(tree)

if __name__ == '__main__':
    main()
<commit_msg>cq python dist<commit_after>"""
Python parser for parsing code into a JSON object
Accepts input from the command-line either as STDIN
or a raw string.

Example usage:

  python src/parser.py "$(cat test/data/bye.py)"
  cat test/data/bye.py | python src/parser.py

Most credit goes to https://github.com/exana/python-ast-explorer
for parsing ideas
"""

import sys
import json
import ast
import io
import tokenize
import token
import collections

Token = collections.namedtuple("Token", "type string lineno col_offset start end")

from pprint import pprint
from argparse import ArgumentParser

TYPE = 0
STRING = 1
LINENO = 2
COL_OFFSET = 3
START = 4
END = 5

def fix_ast(tree, source_lines, tokens):
    def match_tokens(node):
        """
        Match tokens with this node
        """
        lineno, col_offset = node.lineno, node.col_offset

        # Make sure you filter out tokens we're not interested in, particularly
        # DEDENTs There are probably other types we'll come across that need
        # filtered out, so you can find the list here:
        # https://docs.python.org/2/library/token.html#token.ENDMARKER
        possible_tokens = list(filter((lambda tok: tok[LINENO] == lineno
                                       and tok[COL_OFFSET] == col_offset
                                       and tok[TYPE] not in [token.DEDENT]), tokens))

        # print possible_tokens
        if len(possible_tokens) > 0:
            tok = possible_tokens.pop(0)
            node.string = tok[STRING]
            node.start = tok[START]
            node.end = tok[END]

    # TODO - combine both of these steps into a single breadth first search.
    # what we need to change is
    # if a token t1 has a sibling t2, then the end of t1 is the beginning of t2-1

    for node in ast.walk(tree):
        if hasattr(node, 'lineno') and hasattr(node, 'col_offset'):
            match_tokens(node)

    def fix_ends(node, level=0):
        nodeEnd = node.end if hasattr(node, 'end') else None

        for field, value in ast.iter_fields(node):
             if isinstance(value, list):
                for item in value:
                    if isinstance(item, ast.AST):
                        fix_ends(item, level=level+1)
                        nodeEnd = max(nodeEnd, item.end) if hasattr(item, 'end') else nodeEnd
             elif isinstance(value, ast.AST):
                fix_ends(value, level=level+1)
             nodeEnd = max(nodeEnd, value.end) if hasattr(value, 'end') else nodeEnd

        node.end = nodeEnd

    fix_ends(tree)
    return tree

def tokenize_with_char_offsets(source):
    readline = io.StringIO(source).readline
    encoding = 'utf8'
    token_source = tokenize.generate_tokens(readline)
    char_lines = list(map(lambda line: line + "\n", source.split("\n")))
    line_lens = [ len(line) for line in char_lines ]

    # pprint(['char_lines', char_lines])
    # pprint(['line_lens', line_lens])

    byte_lines = None
    tokens = []

    for token in token_source:
        # pprint('\n')
        # pprint(['token', token])
        # if token[TYPE] == encoding:
            # first token
            # encoding = token[STRING]
        byte_lines = list(map(lambda line: line.encode(encoding), char_lines))

        if token[LINENO][0] == 0 or (token[LINENO][1] == 0 and token[COL_OFFSET][1] == 0):
            # q: what's even the point of these tokens?
            # a: at least DEDENT tokens. they confuse the future steps though, so filter them out downstream
            tok = Token(token[TYPE], token[STRING],
                        token[LINENO][0], token[LINENO][1],
                        token[COL_OFFSET][0], token[COL_OFFSET][1])

        else:
            assert token[LINENO][0] > 0 # lineno is > 0

            start_line = token[LINENO][0]  # closed [ e.g. includes
            start_char = token[LINENO][1]
            end_line = token[COL_OFFSET][0]
            end_char = token[COL_OFFSET][1] # open ) (e.g. minus 1 to include)

            byte_start_line = byte_lines[start_line-1]
            char_start_col = len(byte_start_line[:start_char].decode(encoding))

            byte_end_line = byte_lines[token[COL_OFFSET][0]-1]
            char_end_col = len(byte_end_line[:token[COL_OFFSET][1]].decode(encoding))

            # pprint(['start_line', start_line, range(1, start_line)])

            start_lines_offset = reduce(lambda x, y: x + line_lens[y-1], range(1, start_line), 0)
            end_lines_offset = reduce(lambda x, y: x + line_lens[y-1], range(1, end_line), 0)

            # start position = sum(prevlines) + starting_char
            # end position = sum(prevlines) +  sum(ending_char)

            # bug in start_lines_offset
            # pprint(['start_lines_offset', 'char_start_col', start_lines_offset, char_start_col])
            # pprint(['end_lines_offset', 'char_end_col', end_lines_offset, char_end_col])

            start_char_offset = start_lines_offset + char_start_col
            end_char_offset = end_lines_offset + char_end_col

            tok = Token(token[TYPE], token[STRING],
                        start_line, start_char,
                        start_char_offset, end_char_offset)

            # pprint(tok)

        # pprint(tok)
        tokens.append(tok)

    return tokens

def parse_ast(node, tokens, code=None):
    """
    Parse the ast into a reasonable JSON object
    """
    if isinstance(node, ast.Module):
        # Just starting out
        result = {
            "type": "Program",
            "start": 0,
            "end": len(code), # hmm
            "body": parse_ast(node.body, tokens)
        }
        return result
    # node or list?
    if isinstance(node, list):
        result = []
        for child_node in node:
            result += [parse_ast(child_node, tokens)]
        return result

    if isinstance(node, ast.AST):
        result = {"type": node.__class__.__name__}
        for k in node.__dict__:
            val = getattr(node, k)
            # if isinstance(val, ast.Str):
                # result["id"] = {"type": "Identifier", "name": node.value.s}
                # result[k] = parse_ast(val, tokens)
            # else:
            result[k] = parse_ast(val, tokens)

        return result

    return node

def make_ast(code, mode='exec'):
    """
    Make an AST tree from a string of code
    """
    code = unicode(code)
    tree = ast.parse(code, '<unknown>', mode)
    tokens = tokenize_with_char_offsets(code)
    tree = fix_ast(tree, code, tokens)
    return parse_ast(tree, tokens, code)

def read_from_stdin():
    """
    Read from stdin helper
    """
    while True:
        output = sys.stdin.read()
        if not output:
            print '[No more data]'
            break
        return output

def main():
    """
    Main
    """
    cli_parser = ArgumentParser(
        description="Convert python code into a JSON tree")
    cli_parser.add_argument(
        'text', nargs='?', default=None, action='store', help='Code to parse')
    args = cli_parser.parse_args()

    code = args.text if args.text is not None else read_from_stdin()
    tree = make_ast(code)

    print json.dumps(tree)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from typing import List, Optional

__version__ = "21.1"

def main(args=None):
    # type: (Optional[List[str]]) -> int
    """This is an internal API only meant for use by pip's own console scripts.

    For additional details, see https://github.com/pypa/pip/issues/7498.
    """
    from pip._internal.utils.entrypoints import _wrapper

    return _wrapper(args)
<commit_msg>Bump for development<commit_after>from typing import List, Optional

__version__ = "21.2.dev0"

def main(args=None):
    # type: (Optional[List[str]]) -> int
    """This is an internal API only meant for use by pip's own console scripts.

    For additional details, see https://github.com/pypa/pip/issues/7498.
    """
    from pip._internal.utils.entrypoints import _wrapper

    return _wrapper(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
tests a test set of data using a specified, pre0trained model and weights
"""
from __future__ import absolute_import, division, print_function
from ibeis_cnn import utils
from ibeis_cnn import models

import cPickle as pickle
from lasagne import layers

import time
import utool as ut
import six  # NOQA
from os.path import join, abspath

def test(data_fpath, model, weights_fpath, results_dpath, labels_fpath=None, **kwargs):
    """
    Driver function

    Args:
        data_fpath (?):
        labels_fpath (?):
        model (?):
        weights_fpath (?):
    """

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    print('data_fpath = %r' % (data_fpath,))
    X_test, y_test = utils.load(data_fpath, labels_fpath)

    # Load the pretrained model if specified
    print('[model] loading pretrained weights from %s' % (weights_fpath))
    pretrained_weights = None
    with open(weights_fpath, 'rb') as pfile:
        kwargs = pickle.load(pfile)
        pretrained_weights = kwargs.pop('best_weights', None)

    print('test kwargs = \n' + (ut.dict_str(kwargs)))

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_height, input_width, input_channels = kwargs.get('model_shape', None)  # SHOULD ERROR IF NOT SET
    output_layer = model.build_model(
        kwargs.get('batch_size'), input_width, input_height,
        input_channels, kwargs.get('output_dims'))
    utils.print_layer_info(output_layer)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    test_iter, test_iter_accuracy = utils.create_testing_func(output_layer, **kwargs)

    # Set weights to model
    layers.set_all_param_values(output_layer, pretrained_weights)

    # Begin testing with the neural network
    print('\n[test] starting testing with batch size %0.1f' % (kwargs.get('batch_size'), ))

    # Start timer
    t0 = time.time()

    all_predict, labels = utils.forward_test_predictions(X_test, test_iter, results_dpath, **kwargs)

    if y_test is not None:
        mapping_fn = getattr(model, 'label_order_mapping', None)
        avg_test_accuracy = utils.forward_test(X_test, y_test, test_iter_accuracy,
                                               results_dpath, mapping_fn, **kwargs)
        print('Test accuracy for %d examples: %0.2f' % (len(X_test), avg_test_accuracy, ))

    # End timer
    t1 = time.time()
    print('\n[test] prediction took %0.2f seconds' % (t1 - t0, ))

def display_caffe_model(weights_model_path, results_path, **kwargs):
    """
    Driver function

    Args:
        data_fpath (?):
        labels_fpath (?):
        model (?):
        weights_fpath (?):
    """

    ######################################################################################

    # Load the pretrained model if specified
    print('[model] loading pretrained weights and model from %s' % (weights_model_path))
    pretrained_weights = None
    with open(weights_model_path, 'rb') as pfile:
        pretrained_weights = pickle.load(pfile)

    utils.show_convolutional_features(pretrained_weights, results_path, color=False, target=0)

def test_pz():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz()
    """
    project_name            = 'plains'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_large():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_large

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_large()
    """
    project_name            = 'plains_large'
    model                   = models.PZ_GIRM_LARGE_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_girm():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_girm

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_girm()
    """
    project_name            = 'viewpoint'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_girm_large():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_girm_large

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_girm_large()
    """
    project_name            = 'viewpoint_large'
    model                   = models.PZ_GIRM_LARGE_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_display_caffenet():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_display_caffenet

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_display_caffenet()
    """
    project_name            = 'caffenet'

    root                    = abspath(join('..', 'data'))
    results_dpath           = join(root, 'results', project_name)
    weights_model_fpath     = join(root, 'nets', project_name, 'caffenet3.caffe.pickle')
    config = {}

    display_caffe_model(weights_model_fpath, results_dpath, **config)

if __name__ == '__main__':
    """
    CommandLine:
        python -m ibeis_cnn.test
        python -m ibeis_cnn.test --allexamples
        python -m ibeis_cnn.test --allexamples --noface --nosrc

    CommandLine:
        cd %CODE_DIR%/ibies_cnn/code
        cd $CODE_DIR/ibies_cnn/code
        code
        cd ibeis_cnn/code
        python test.py

    PythonPrereqs:
        pip install theano
        pip install git+https://github.com/Lasagne/Lasagne.git
        pip install git+git://github.com/lisa-lab/pylearn2.git
        #pip install lasagne
        #pip install pylearn2
        git clone git://github.com/lisa-lab/pylearn2.git
        git clone https://github.com/Lasagne/Lasagne.git
        cd pylearn2
        python setup.py develop
        cd ..
        cd Lesagne
        git checkout 8758ac1434175159e5c1f30123041799c2b6098a
        python setup.py develop
    """
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()
<commit_msg>Caffenet displayer<commit_after>#!/usr/bin/env python
"""
tests a test set of data using a specified, pre0trained model and weights
"""
from __future__ import absolute_import, division, print_function
from ibeis_cnn import utils
from ibeis_cnn import models

import cPickle as pickle
from lasagne import layers

import time
import utool as ut
import six  # NOQA
from os.path import join, abspath

def test(data_fpath, model, weights_fpath, results_dpath, labels_fpath=None, **kwargs):
    """
    Driver function

    Args:
        data_fpath (?):
        labels_fpath (?):
        model (?):
        weights_fpath (?):
    """

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    print('data_fpath = %r' % (data_fpath,))
    X_test, y_test = utils.load(data_fpath, labels_fpath)

    # Load the pretrained model if specified
    print('[model] loading pretrained weights from %s' % (weights_fpath))
    pretrained_weights = None
    with open(weights_fpath, 'rb') as pfile:
        kwargs = pickle.load(pfile)
        pretrained_weights = kwargs.pop('best_weights', None)

    print('test kwargs = \n' + (ut.dict_str(kwargs)))

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_height, input_width, input_channels = kwargs.get('model_shape', None)  # SHOULD ERROR IF NOT SET
    output_layer = model.build_model(
        kwargs.get('batch_size'), input_width, input_height,
        input_channels, kwargs.get('output_dims'))
    utils.print_layer_info(output_layer)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    test_iter, test_iter_accuracy = utils.create_testing_func(output_layer, **kwargs)

    # Set weights to model
    layers.set_all_param_values(output_layer, pretrained_weights)

    # Begin testing with the neural network
    print('\n[test] starting testing with batch size %0.1f' % (kwargs.get('batch_size'), ))

    # Start timer
    t0 = time.time()

    all_predict, labels = utils.forward_test_predictions(X_test, test_iter, results_dpath, **kwargs)

    if y_test is not None:
        mapping_fn = getattr(model, 'label_order_mapping', None)
        avg_test_accuracy = utils.forward_test(X_test, y_test, test_iter_accuracy,
                                               results_dpath, mapping_fn, **kwargs)
        print('Test accuracy for %d examples: %0.2f' % (len(X_test), avg_test_accuracy, ))

    # End timer
    t1 = time.time()
    print('\n[test] prediction took %0.2f seconds' % (t1 - t0, ))

def display_caffe_model(weights_model_path, results_path, **kwargs):
    """
    Driver function

    Args:
        data_fpath (?):
        labels_fpath (?):
        model (?):
        weights_fpath (?):
    """

    ######################################################################################

    # Load the pretrained model if specified
    print('[model] loading pretrained weights and model from %s' % (weights_model_path))
    pretrained_weights = None
    with open(weights_model_path, 'rb') as pfile:
        pretrained_weights = pickle.load(pfile)

    utils.show_convolutional_features(pretrained_weights, results_path, color=False, target=0)
    utils.show_convolutional_features(pretrained_weights, results_path, color=True, target=0)

def test_pz():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz()
    """
    project_name            = 'plains'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_large():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_large

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_large()
    """
    project_name            = 'plains_large'
    model                   = models.PZ_GIRM_LARGE_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_girm():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_girm

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_girm()
    """
    project_name            = 'viewpoint'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_pz_girm_large():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_pz_girm_large

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_pz_girm_large()
    """
    project_name            = 'viewpoint_large'
    model                   = models.PZ_GIRM_LARGE_Model()

    root                    = abspath(join('..', 'data'))
    test_data_fpath         = join(root, 'numpy', project_name, 'X.npy')
    test_labels_fpath       = join(root, 'numpy', project_name, 'y.npy')
    results_dpath           = join(root, 'results', project_name)
    weights_fpath           = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    config = {}

    test(test_data_fpath, model, weights_fpath, results_dpath, test_labels_fpath, **config)

def test_display_caffenet():
    r"""
    CommandLine:
        python -m ibeis_cnn.test --test-test_display_caffenet

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.test import *  # NOQA
        >>> test_display_caffenet()
    """
    project_name            = 'caffenet'

    root                    = abspath(join('..', 'data'))
    results_dpath           = join(root, 'results', project_name)
    weights_model_fpath     = join(root, 'nets', project_name, 'caffenet3.caffe.pickle')
    config = {}

    display_caffe_model(weights_model_fpath, results_dpath, **config)

if __name__ == '__main__':
    """
    CommandLine:
        python -m ibeis_cnn.test
        python -m ibeis_cnn.test --allexamples
        python -m ibeis_cnn.test --allexamples --noface --nosrc

    CommandLine:
        cd %CODE_DIR%/ibies_cnn/code
        cd $CODE_DIR/ibies_cnn/code
        code
        cd ibeis_cnn/code
        python test.py

    PythonPrereqs:
        pip install theano
        pip install git+https://github.com/Lasagne/Lasagne.git
        pip install git+git://github.com/lisa-lab/pylearn2.git
        #pip install lasagne
        #pip install pylearn2
        git clone git://github.com/lisa-lab/pylearn2.git
        git clone https://github.com/Lasagne/Lasagne.git
        cd pylearn2
        python setup.py develop
        cd ..
        cd Lesagne
        git checkout 8758ac1434175159e5c1f30123041799c2b6098a
        python setup.py develop
    """
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()
<|endoftext|>


<commit_before>"""

>>> from blaze.expr import Symbol
>>> from blaze.compute.pandas import compute

>>> accounts = Symbol('accounts', 'var * {name: string, amount: int}')
>>> deadbeats = accounts[accounts['amount'] < 0]['name']

>>> from pandas import DataFrame
>>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
>>> df = DataFrame(data, columns=['name', 'amount'])
>>> compute(deadbeats, df)
1        Bob
2    Charlie
Name: name, dtype: object
"""
from __future__ import absolute_import, division, print_function

import pandas as pd
from pandas.core.generic import NDFrame
from pandas import DataFrame, Series
from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy
import numpy as np
from collections import defaultdict
from toolz import merge as merge_dicts
import fnmatch
from datashape.predicates import isscalar

from ..api.into import into
from ..dispatch import dispatch
from ..expr import (Projection, Field, Sort, Head, Broadcast, Selection,
                    Reduction, Distinct, Join, By, Summary, Label, ReLabel,
                    Map, Apply, Merge, Union, std, var, Like, Slice,
                    ElemWise, DateTime, Millisecond, Expr, Symbol,
                    UTCFromTimestamp, nelements)
from ..expr import UnaryOp, BinOp
from ..expr import Symbol, common_subexpression
from .core import compute, compute_up, base
from ..compatibility import _inttypes

__all__ = []

@dispatch(Projection, DataFrame)
def compute_up(t, df, **kwargs):
    return df[list(t.fields)]

@dispatch(Field, (DataFrame, DataFrameGroupBy))
def compute_up(t, df, **kwargs):
    return df[t.fields[0]]

@dispatch(Field, Series)
def compute_up(t, data, **kwargs):
    if t.fields[0] == data.name:
        return data
    else:
        raise ValueError("Fieldname %s does not match Series name %s"
                % (t.fields[0], data.name))

@dispatch(Broadcast, DataFrame)
def compute_up(t, df, **kwargs):
    d = dict((t._child[c]._expr, df[c]) for c in t._child.fields)
    return compute(t._expr, d)

@dispatch(Broadcast, Series)
def compute_up(t, s, **kwargs):
    return compute_up(t, s.to_frame(), **kwargs)

@dispatch(BinOp, Series)
def compute_up(t, data, **kwargs):
    if isinstance(t.lhs, Expr):
        return t.op(data, t.rhs)
    else:
        return t.op(t.lhs, data)

@dispatch(BinOp, Series, (Series, base))
def compute_up(t, lhs, rhs, **kwargs):
    return t.op(lhs, rhs)

@dispatch(BinOp, (Series, base), Series)
def compute_up(t, lhs, rhs, **kwargs):
    return t.op(lhs, rhs)

@dispatch(UnaryOp, NDFrame)
def compute_up(t, df, **kwargs):
    f = getattr(t, 'op', getattr(np, t.symbol, None))
    if f is None:
        raise ValueError('%s is not a valid operation on %s objects' %
                         (t.symbol, type(df).__name__))
    return f(df)

@dispatch(Selection, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    predicate = compute(t.predicate, {t._child: df})
    return df[predicate]

@dispatch(Symbol, DataFrame)
def compute_up(t, df, **kwargs):
    if not list(t.fields) == list(df.names):
        # TODO also check dtype
        raise ValueError("Schema mismatch: \n\nTable:\n%s\n\nDataFrame:\n%s"
                         % (t, df))
    return df

@dispatch(Join, DataFrame, DataFrame)
def compute_up(t, lhs, rhs, **kwargs):
    """ Join two pandas data frames on arbitrary columns

    The approach taken here could probably be improved.

    To join on two columns we force each column to be the index of the
    dataframe, perform the join, and then reset the index back to the left
    side's original index.
    """
    result = pd.merge(lhs, rhs,
                      left_on=t.on_left, right_on=t.on_right,
                      how=t.how)
    return result.reset_index()[t.fields]

@dispatch(Symbol, (DataFrameGroupBy, SeriesGroupBy))
def compute_up(t, gb, **kwargs):
    return gb

def post_reduction(result):
    # pandas may return an int, numpy scalar or non scalar here so we need to
    # program defensively so that things are JSON serializable
    try:
        return result.item()
    except (AttributeError, ValueError):
        return result

@dispatch(Reduction, (Series, SeriesGroupBy))
def compute_up(t, s, **kwargs):
    result = post_reduction(getattr(s, t.symbol)())
    if t.keepdims:
        result = Series([result], name=s.name)
    return result

@dispatch((std, var), (Series, SeriesGroupBy))
def compute_up(t, s, **kwargs):
    result = post_reduction(getattr(s, t.symbol)(ddof=t.unbiased))
    if t.keepdims:
        result = Series([result], name=s.name)
    return result

@dispatch(Distinct, DataFrame)
def compute_up(t, df, **kwargs):
    return df.drop_duplicates()

@dispatch(Distinct, Series)
def compute_up(t, s, **kwargs):
    s2 = Series(s.unique())
    s2.name = s.name
    return s2

def unpack(seq):
    """ Unpack sequence of length one

    >>> unpack([1, 2, 3])
    [1, 2, 3]

    >>> unpack([1])
    1
    """
    seq = list(seq)
    if len(seq) == 1:
        seq = seq[0]
    return seq

Grouper = ElemWise, Series, list

@dispatch(By, list, DataFrame)
def get_grouper(c, grouper, df):
    return grouper

@dispatch(By, (ElemWise, Series), NDFrame)
def get_grouper(c, grouper, df):
    return compute(grouper, {c._child: df})

@dispatch(By, (Field, Projection), NDFrame)
def get_grouper(c, grouper, df):
    return grouper.fields

@dispatch(By, Reduction, Grouper, NDFrame)
def compute_by(t, r, g, df):
    names = [r._name]
    preapply = compute(r._child, {t._child: df})

    # Pandas and Blaze column naming schemes differ
    # Coerce DataFrame column names to match Blaze's names
    preapply = preapply.copy()
    if isinstance(preapply, Series):
        preapply.name = names[0]
    else:
        preapply.names = names
    group_df = concat_nodup(df, preapply)

    gb = group_df.groupby(g)
    groups = gb[names[0] if isscalar(t.apply._child.dshape.measure) else names]

    return compute_up(r, groups)  # do reduction

@dispatch(By, Summary, Grouper, NDFrame)
def compute_by(t, s, g, df):
    names = s.fields
    preapply = DataFrame(dict(zip(names,
                                  (compute(v._child, {t._child: df})
                                   for v in s.values))))

    df2 = concat_nodup(df, preapply)

    groups = df2.groupby(g)

    d = defaultdict(list)
    for name, v in zip(names, s.values):
        d[name].append(getattr(Series, v.symbol))

    result = groups.agg(dict(d))

    # Rearrange columns to match names order
    result = result[sorted(result.columns, key=lambda t: names.index(t[0]))]
    result.columns = t.apply.fields  # flatten down multiindex
    return result

@dispatch(Expr, DataFrame)
def post_compute_by(t, df):
    return df.reset_index(drop=True)

@dispatch((Summary, Reduction), DataFrame)
def post_compute_by(t, df):
    return df.reset_index()

@dispatch(By, NDFrame)
def compute_up(t, df, **kwargs):
    grouper = get_grouper(t, t.grouper, df)
    result = compute_by(t, t.apply, grouper, df)
    return post_compute_by(t.apply, into(DataFrame, result))

def concat_nodup(a, b):
    """ Concatenate two dataframes/series without duplicately named columns

    >>> df = DataFrame([[1, 'Alice',   100],
    ...                 [2, 'Bob',    -200],
    ...                 [3, 'Charlie', 300]],
    ...                columns=['id','name', 'amount'])

    >>> concat_nodup(df, df)
       id     name  amount
    0   1    Alice     100
    1   2      Bob    -200
    2   3  Charlie     300

    >>> concat_nodup(df.name, df.amount)
          name  amount
    0    Alice     100
    1      Bob    -200
    2  Charlie     300

    >>> concat_nodup(df, df.amount + df.id)
       id     name  amount    0
    0   1    Alice     100  101
    1   2      Bob    -200 -198
    2   3  Charlie     300  303
    """

    if isinstance(a, DataFrame) and isinstance(b, DataFrame):
        return pd.concat([a, b[[c for c in b.columns if c not in a.columns]]],
                         axis=1)
    if isinstance(a, DataFrame) and isinstance(b, Series):
        if b.name not in a.columns:
            return pd.concat([a, b], axis=1)
        else:
            return a
    if isinstance(a, Series) and isinstance(b, DataFrame):
        return pd.concat([a, b[[c for c in b.columns if c != a.name]]], axis=1)
    if isinstance(a, Series) and isinstance(b, Series):
        if a.name == b.name:
            return a
        else:
            return pd.concat([a, b], axis=1)

@dispatch(Sort, DataFrame)
def compute_up(t, df, **kwargs):
    return df.sort(t.key, ascending=t.ascending)

@dispatch(Sort, Series)
def compute_up(t, s, **kwargs):
    return s.order(ascending=t.ascending)

@dispatch(Head, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    return df.head(t.n)

@dispatch(Label, DataFrame)
def compute_up(t, df, **kwargs):
    return DataFrame(df, columns=[t.label])

@dispatch(Label, Series)
def compute_up(t, df, **kwargs):
    return Series(df, name=t.label)

@dispatch(ReLabel, DataFrame)
def compute_up(t, df, **kwargs):
    return df.rename(columns=dict(t.labels))

@dispatch(ReLabel, Series)
def compute_up(t, s, **kwargs):
    labels = t.labels
    if len(labels) > 1:
        raise ValueError('You can only relabel a Series with a single name')
    pair, = labels
    _, replacement = pair
    return Series(s, name=replacement)

@dispatch(Map, DataFrame)
def compute_up(t, df, **kwargs):
    return df.apply(lambda tup: t.func(*tup), axis=1)

@dispatch(Map, Series)
def compute_up(t, df, **kwargs):
    result = df.map(t.func)
    try:
        result.name = t._name
    except NotImplementedError:
        # We don't have a schema, but we should still be able to map
        result.name = df.name
    return result

@dispatch(Apply, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    return t.func(df)

@dispatch(Merge, NDFrame)
def compute_up(t, df, scope=None, **kwargs):
    subexpression = common_subexpression(*t.children)
    scope = merge_dicts(scope or {}, {subexpression: df})
    children = [compute(_child, scope) for _child in t.children]
    return pd.concat(children, axis=1)

@dispatch(Union, (Series, DataFrame), tuple)
def compute_up(t, example, children, **kwargs):
    return pd.concat(children, axis=0)

@dispatch(Summary, DataFrame)
def compute_up(expr, data, **kwargs):
    values = [compute(val, {expr._child: data}) for val in expr.values]
    if expr.keepdims:
        return DataFrame([values], columns=expr.fields)
    else:
        return Series(dict(zip(expr.fields, values)))

@dispatch(Like, DataFrame)
def compute_up(expr, df, **kwargs):
    arrs = [df[name].str.contains('^%s$' % fnmatch.translate(pattern))
            for name, pattern in expr.patterns.items()]
    return df[np.logical_and.reduce(arrs)]

def get_date_attr(s, attr):
    try:
        # new in pandas 0.15
        return getattr(s.dt, attr)
    except AttributeError:
        return getattr(pd.DatetimeIndex(s), attr)

@dispatch(DateTime, Series)
def compute_up(expr, s, **kwargs):
    return get_date_attr(s, expr.attr)

@dispatch(UTCFromTimestamp, Series)
def compute_up(expr, s, **kwargs):
    return pd.datetools.to_datetime(s*1e9, utc=True)

@dispatch(Millisecond, Series)
def compute_up(_, s, **kwargs):
    return get_date_attr(s, 'microsecond') // 1000

@dispatch(Slice, (DataFrame, Series))
def compute_up(expr, df, **kwargs):
    index = expr.index
    if isinstance(index, tuple) and len(index) == 1:
        index = index[0]
    if isinstance(index, _inttypes):
        return df.iloc[index]
    elif isinstance(index, slice):
        if index.stop is not None:
            return df.iloc[index.start:index.stop:index.step]
        else:
            return df.iloc[index]
    else:
        raise NotImplementedError()

@dispatch(nelements, (DataFrame, Series))
def compute_up(expr, df, **kwargs):
    return np.prod([df.shape[i] for i in (expr.axis or expr.ndim)])
<commit_msg>No parens!<commit_after>"""

>>> from blaze.expr import Symbol
>>> from blaze.compute.pandas import compute

>>> accounts = Symbol('accounts', 'var * {name: string, amount: int}')
>>> deadbeats = accounts[accounts['amount'] < 0]['name']

>>> from pandas import DataFrame
>>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
>>> df = DataFrame(data, columns=['name', 'amount'])
>>> compute(deadbeats, df)
1        Bob
2    Charlie
Name: name, dtype: object
"""
from __future__ import absolute_import, division, print_function

import pandas as pd
from pandas.core.generic import NDFrame
from pandas import DataFrame, Series
from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy
import numpy as np
from collections import defaultdict
from toolz import merge as merge_dicts
import fnmatch
from datashape.predicates import isscalar

from ..api.into import into
from ..dispatch import dispatch
from ..expr import (Projection, Field, Sort, Head, Broadcast, Selection,
                    Reduction, Distinct, Join, By, Summary, Label, ReLabel,
                    Map, Apply, Merge, Union, std, var, Like, Slice,
                    ElemWise, DateTime, Millisecond, Expr, Symbol,
                    UTCFromTimestamp, nelements)
from ..expr import UnaryOp, BinOp
from ..expr import Symbol, common_subexpression
from .core import compute, compute_up, base
from ..compatibility import _inttypes

__all__ = []

@dispatch(Projection, DataFrame)
def compute_up(t, df, **kwargs):
    return df[list(t.fields)]

@dispatch(Field, (DataFrame, DataFrameGroupBy))
def compute_up(t, df, **kwargs):
    return df[t.fields[0]]

@dispatch(Field, Series)
def compute_up(t, data, **kwargs):
    if t.fields[0] == data.name:
        return data
    else:
        raise ValueError("Fieldname %s does not match Series name %s"
                % (t.fields[0], data.name))

@dispatch(Broadcast, DataFrame)
def compute_up(t, df, **kwargs):
    d = dict((t._child[c]._expr, df[c]) for c in t._child.fields)
    return compute(t._expr, d)

@dispatch(Broadcast, Series)
def compute_up(t, s, **kwargs):
    return compute_up(t, s.to_frame(), **kwargs)

@dispatch(BinOp, Series)
def compute_up(t, data, **kwargs):
    if isinstance(t.lhs, Expr):
        return t.op(data, t.rhs)
    else:
        return t.op(t.lhs, data)

@dispatch(BinOp, Series, (Series, base))
def compute_up(t, lhs, rhs, **kwargs):
    return t.op(lhs, rhs)

@dispatch(BinOp, (Series, base), Series)
def compute_up(t, lhs, rhs, **kwargs):
    return t.op(lhs, rhs)

@dispatch(UnaryOp, NDFrame)
def compute_up(t, df, **kwargs):
    f = getattr(t, 'op', getattr(np, t.symbol, None))
    if f is None:
        raise ValueError('%s is not a valid operation on %s objects' %
                         (t.symbol, type(df).__name__))
    return f(df)

@dispatch(Selection, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    predicate = compute(t.predicate, {t._child: df})
    return df[predicate]

@dispatch(Symbol, DataFrame)
def compute_up(t, df, **kwargs):
    if not list(t.fields) == list(df.names):
        # TODO also check dtype
        raise ValueError("Schema mismatch: \n\nTable:\n%s\n\nDataFrame:\n%s"
                         % (t, df))
    return df

@dispatch(Join, DataFrame, DataFrame)
def compute_up(t, lhs, rhs, **kwargs):
    """ Join two pandas data frames on arbitrary columns

    The approach taken here could probably be improved.

    To join on two columns we force each column to be the index of the
    dataframe, perform the join, and then reset the index back to the left
    side's original index.
    """
    result = pd.merge(lhs, rhs,
                      left_on=t.on_left, right_on=t.on_right,
                      how=t.how)
    return result.reset_index()[t.fields]

@dispatch(Symbol, (DataFrameGroupBy, SeriesGroupBy))
def compute_up(t, gb, **kwargs):
    return gb

def post_reduction(result):
    # pandas may return an int, numpy scalar or non scalar here so we need to
    # program defensively so that things are JSON serializable
    try:
        return result.item()
    except (AttributeError, ValueError):
        return result

@dispatch(Reduction, (Series, SeriesGroupBy))
def compute_up(t, s, **kwargs):
    result = post_reduction(getattr(s, t.symbol)())
    if t.keepdims:
        result = Series([result], name=s.name)
    return result

@dispatch((std, var), (Series, SeriesGroupBy))
def compute_up(t, s, **kwargs):
    result = post_reduction(getattr(s, t.symbol)(ddof=t.unbiased))
    if t.keepdims:
        result = Series([result], name=s.name)
    return result

@dispatch(Distinct, DataFrame)
def compute_up(t, df, **kwargs):
    return df.drop_duplicates()

@dispatch(Distinct, Series)
def compute_up(t, s, **kwargs):
    s2 = Series(s.unique())
    s2.name = s.name
    return s2

def unpack(seq):
    """ Unpack sequence of length one

    >>> unpack([1, 2, 3])
    [1, 2, 3]

    >>> unpack([1])
    1
    """
    seq = list(seq)
    if len(seq) == 1:
        seq = seq[0]
    return seq

Grouper = ElemWise, Series, list

@dispatch(By, list, DataFrame)
def get_grouper(c, grouper, df):
    return grouper

@dispatch(By, (ElemWise, Series), NDFrame)
def get_grouper(c, grouper, df):
    return compute(grouper, {c._child: df})

@dispatch(By, (Field, Projection), NDFrame)
def get_grouper(c, grouper, df):
    return grouper.fields

@dispatch(By, Reduction, Grouper, NDFrame)
def compute_by(t, r, g, df):
    names = [r._name]
    preapply = compute(r._child, {t._child: df})

    # Pandas and Blaze column naming schemes differ
    # Coerce DataFrame column names to match Blaze's names
    preapply = preapply.copy()
    if isinstance(preapply, Series):
        preapply.name = names[0]
    else:
        preapply.names = names
    group_df = concat_nodup(df, preapply)

    gb = group_df.groupby(g)
    groups = gb[names[0] if isscalar(t.apply._child.dshape.measure) else names]

    return compute_up(r, groups)  # do reduction

@dispatch(By, Summary, Grouper, NDFrame)
def compute_by(t, s, g, df):
    names = s.fields
    preapply = DataFrame(dict(zip(names,
                                  (compute(v._child, {t._child: df})
                                   for v in s.values))))

    df2 = concat_nodup(df, preapply)

    groups = df2.groupby(g)

    d = defaultdict(list)
    for name, v in zip(names, s.values):
        d[name].append(getattr(Series, v.symbol))

    result = groups.agg(dict(d))

    # Rearrange columns to match names order
    result = result[sorted(result.columns, key=lambda t: names.index(t[0]))]
    result.columns = t.apply.fields  # flatten down multiindex
    return result

@dispatch(Expr, DataFrame)
def post_compute_by(t, df):
    return df.reset_index(drop=True)

@dispatch((Summary, Reduction), DataFrame)
def post_compute_by(t, df):
    return df.reset_index()

@dispatch(By, NDFrame)
def compute_up(t, df, **kwargs):
    grouper = get_grouper(t, t.grouper, df)
    result = compute_by(t, t.apply, grouper, df)
    return post_compute_by(t.apply, into(DataFrame, result))

def concat_nodup(a, b):
    """ Concatenate two dataframes/series without duplicately named columns

    >>> df = DataFrame([[1, 'Alice',   100],
    ...                 [2, 'Bob',    -200],
    ...                 [3, 'Charlie', 300]],
    ...                columns=['id','name', 'amount'])

    >>> concat_nodup(df, df)
       id     name  amount
    0   1    Alice     100
    1   2      Bob    -200
    2   3  Charlie     300

    >>> concat_nodup(df.name, df.amount)
          name  amount
    0    Alice     100
    1      Bob    -200
    2  Charlie     300

    >>> concat_nodup(df, df.amount + df.id)
       id     name  amount    0
    0   1    Alice     100  101
    1   2      Bob    -200 -198
    2   3  Charlie     300  303
    """

    if isinstance(a, DataFrame) and isinstance(b, DataFrame):
        return pd.concat([a, b[[c for c in b.columns if c not in a.columns]]],
                         axis=1)
    if isinstance(a, DataFrame) and isinstance(b, Series):
        if b.name not in a.columns:
            return pd.concat([a, b], axis=1)
        else:
            return a
    if isinstance(a, Series) and isinstance(b, DataFrame):
        return pd.concat([a, b[[c for c in b.columns if c != a.name]]], axis=1)
    if isinstance(a, Series) and isinstance(b, Series):
        if a.name == b.name:
            return a
        else:
            return pd.concat([a, b], axis=1)

@dispatch(Sort, DataFrame)
def compute_up(t, df, **kwargs):
    return df.sort(t.key, ascending=t.ascending)

@dispatch(Sort, Series)
def compute_up(t, s, **kwargs):
    return s.order(ascending=t.ascending)

@dispatch(Head, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    return df.head(t.n)

@dispatch(Label, DataFrame)
def compute_up(t, df, **kwargs):
    return DataFrame(df, columns=[t.label])

@dispatch(Label, Series)
def compute_up(t, df, **kwargs):
    return Series(df, name=t.label)

@dispatch(ReLabel, DataFrame)
def compute_up(t, df, **kwargs):
    return df.rename(columns=dict(t.labels))

@dispatch(ReLabel, Series)
def compute_up(t, s, **kwargs):
    labels = t.labels
    if len(labels) > 1:
        raise ValueError('You can only relabel a Series with a single name')
    pair, = labels
    _, replacement = pair
    return Series(s, name=replacement)

@dispatch(Map, DataFrame)
def compute_up(t, df, **kwargs):
    return df.apply(lambda tup: t.func(*tup), axis=1)

@dispatch(Map, Series)
def compute_up(t, df, **kwargs):
    result = df.map(t.func)
    try:
        result.name = t._name
    except NotImplementedError:
        # We don't have a schema, but we should still be able to map
        result.name = df.name
    return result

@dispatch(Apply, (Series, DataFrame))
def compute_up(t, df, **kwargs):
    return t.func(df)

@dispatch(Merge, NDFrame)
def compute_up(t, df, scope=None, **kwargs):
    subexpression = common_subexpression(*t.children)
    scope = merge_dicts(scope or {}, {subexpression: df})
    children = [compute(_child, scope) for _child in t.children]
    return pd.concat(children, axis=1)

@dispatch(Union, (Series, DataFrame), tuple)
def compute_up(t, example, children, **kwargs):
    return pd.concat(children, axis=0)

@dispatch(Summary, DataFrame)
def compute_up(expr, data, **kwargs):
    values = [compute(val, {expr._child: data}) for val in expr.values]
    if expr.keepdims:
        return DataFrame([values], columns=expr.fields)
    else:
        return Series(dict(zip(expr.fields, values)))

@dispatch(Like, DataFrame)
def compute_up(expr, df, **kwargs):
    arrs = [df[name].str.contains('^%s$' % fnmatch.translate(pattern))
            for name, pattern in expr.patterns.items()]
    return df[np.logical_and.reduce(arrs)]

def get_date_attr(s, attr):
    try:
        # new in pandas 0.15
        return getattr(s.dt, attr)
    except AttributeError:
        return getattr(pd.DatetimeIndex(s), attr)

@dispatch(DateTime, Series)
def compute_up(expr, s, **kwargs):
    return get_date_attr(s, expr.attr)

@dispatch(UTCFromTimestamp, Series)
def compute_up(expr, s, **kwargs):
    return pd.datetools.to_datetime(s*1e9, utc=True)

@dispatch(Millisecond, Series)
def compute_up(_, s, **kwargs):
    return get_date_attr(s, 'microsecond') // 1000

@dispatch(Slice, (DataFrame, Series))
def compute_up(expr, df, **kwargs):
    index = expr.index
    if isinstance(index, tuple) and len(index) == 1:
        index = index[0]
    if isinstance(index, _inttypes):
        return df.iloc[index]
    elif isinstance(index, slice):
        if index.stop is not None:
            return df.iloc[index.start:index.stop:index.step]
        else:
            return df.iloc[index]
    else:
        raise NotImplementedError()

@dispatch(nelements, (DataFrame, Series))
def compute_up(expr, df, **kwargs):
    return np.prod([df.shape[i] for i in expr.axis or expr.ndim])
<|endoftext|>


<commit_before># Copyright 2021 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

""" Copyright 2012, 2013 UW Information Technology, University of Washington

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

    Changes
    =================================================================

    sbutler1@illinois.edu: adapt to the new RESTDispatch framework.
"""
import time

from spotseeker_server.views.rest_dispatch import \
    RESTDispatch, RESTException, JSONResponse
from django.http import HttpResponse
from django.utils.http import http_date
from wsgiref.util import FileWrapper
from django.core.exceptions import ValidationError
from django.core.files.images import ImageFile
from spotseeker_server.require_auth import *
from spotseeker_server.models import *

class ItemImageView(RESTDispatch):
    """ Handles actions at /api/v1/item/<item id>/image/<image id>.
    GET returns 200 with the image.
    PUT returns 200 and updates the image.
    DELETE returns 200 and deletes the image.
    """
    @app_auth_required
    def GET(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException("Image Spot ID doesn't match item id in url",
                                404)

        response = HttpResponse(FileWrapper(img.image))
        response["ETag"] = img.etag

        # 7 day timeout?
        response['Expires'] = http_date(time.time() + 60 * 60 * 24 * 7)
        response["Content-length"] = img.image.size
        response["Content-type"] = img.content_type
        return response

    @user_auth_required
    @admin_auth_required
    def PUT(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException("Image Spot ID doesn't match spot id in url",
                                404)

        self.validate_etag(request, img)

        request.method = "POST"
        request._load_post_and_files()
        request.method = "PUT"

        if "image" in request.META['files']:
            img.image = ImageFile(request.META['files']["image"])
        if "description" in request.META['files']:
            img.description = request.META['files']["description"]
        if "display_index" in request.META['files']:
            img.display_index = request.META['files']["display_index"]
        img.save()
        item.spot.save()

        return self.GET(request, item_id, image_id)

    @user_auth_required
    @admin_auth_required
    def DELETE(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException("Image Item ID doesn't match item id in url",
                                404)

        img.delete()
        item.spot.save()

        return HttpResponse(status=200)
<commit_msg>Reformat and update copyright.<commit_after># Copyright 2021 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

""" Changes
    =================================================================

    sbutler1@illinois.edu: adapt to the new RESTDispatch framework.
"""
import time

from spotseeker_server.views.rest_dispatch import (
    RESTDispatch,
    RESTException,
    JSONResponse,
)
from django.http import HttpResponse
from django.utils.http import http_date
from wsgiref.util import FileWrapper
from django.core.exceptions import ValidationError
from django.core.files.images import ImageFile
from spotseeker_server.require_auth import *
from spotseeker_server.models import *

class ItemImageView(RESTDispatch):
    """Handles actions at /api/v1/item/<item id>/image/<image id>.
    GET returns 200 with the image.
    PUT returns 200 and updates the image.
    DELETE returns 200 and deletes the image.
    """

    @app_auth_required
    def GET(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException(
                "Image Spot ID doesn't match item id in url", 404
            )

        response = HttpResponse(FileWrapper(img.image))
        response["ETag"] = img.etag

        # 7 day timeout?
        response["Expires"] = http_date(time.time() + 60 * 60 * 24 * 7)
        response["Content-length"] = img.image.size
        response["Content-type"] = img.content_type
        return response

    @user_auth_required
    @admin_auth_required
    def PUT(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException(
                "Image Spot ID doesn't match spot id in url", 404
            )

        self.validate_etag(request, img)

        request.method = "POST"
        request._load_post_and_files()
        request.method = "PUT"

        if "image" in request.META["files"]:
            img.image = ImageFile(request.META["files"]["image"])
        if "description" in request.META["files"]:
            img.description = request.META["files"]["description"]
        if "display_index" in request.META["files"]:
            img.display_index = request.META["files"]["display_index"]
        img.save()
        item.spot.save()

        return self.GET(request, item_id, image_id)

    @user_auth_required
    @admin_auth_required
    def DELETE(self, request, item_id, image_id):
        img = ItemImage.objects.get(pk=image_id)
        item = img.item

        if int(item.pk) != int(item_id):
            raise RESTException(
                "Image Item ID doesn't match item id in url", 404
            )

        img.delete()
        item.spot.save()

        return HttpResponse(status=200)
<|endoftext|>


<commit_before>from pip._internal.utils.typing import MYPY_CHECK_RUNNING

if MYPY_CHECK_RUNNING:
    from typing import List, Optional

__version__ = "20.2b1"

def main(args=None):
    # type: (Optional[List[str]]) -> int
    """This is an internal API only meant for use by pip's own console scripts.

    For additional details, see https://github.com/pypa/pip/issues/7498.
    """
    from pip._internal.utils.entrypoints import _wrapper

    return _wrapper(args)
<commit_msg>Bump for development<commit_after>from pip._internal.utils.typing import MYPY_CHECK_RUNNING

if MYPY_CHECK_RUNNING:
    from typing import List, Optional

__version__ = "20.2.dev1"

def main(args=None):
    # type: (Optional[List[str]]) -> int
    """This is an internal API only meant for use by pip's own console scripts.

    For additional details, see https://github.com/pypa/pip/issues/7498.
    """
    from pip._internal.utils.entrypoints import _wrapper

    return _wrapper(args)
<|endoftext|>


<commit_before>'''
File: integration.py
Authors:
 - Sjoerd Wenker, 10617558
 - Tristan van Vaalen, 10551832

Contains Euler and second and fourth order Runge-Kutta integration methods.
'''<commit_msg>Euler class, needs refinement<commit_after>'''
File: integration.py
Authors:
 - Sjoerd Wenker, 10617558
 - Tristan van Vaalen, 10551832

Contains Euler and second and fourth order Runge-Kutta integration methods.
'''

from math import isinf

'''
'''
class Euler:
    '''
    Constructor, takes a function, initial timestep, initial y value.
    Optional: stepsize
    '''
    def __init__(self, function, time, y0, stepsize=1):
        self._function = function
        self._timestep = time
        self._x_values = [0] * (time + 1)
        self._y_values = [0] * time + [y0]
        self._stepsize = stepsize

        print self

    '''
    Returns the current list of x values
    '''
    def getXValues(self):
        return self._x_values

    '''
    Returns the current list of y values
    '''
    def getYValues(self):
        return self._y_values

    '''
    Generates the next N values of the given function
    '''
    def generateN(self, N):
        for n in range(N):
            self.getNext()

    '''
    Generates the next value of the given function
    '''
    def getNext(self):
        An = self._function(self._x_values[self._timestep],
                            self._y_values[self._timestep])
        self._x_values.append(self._x_values[self._timestep] + self._stepsize)
        self._y_values.append(self._y_values[self._timestep] +
                              self._stepsize * An)

        if isinf(self._y_values[-1]):
            raise OverflowError(
                "y reached infinity. Stopping generation at {}".format(
                    self._timestep
                )
            )

        self._timestep += 1

    '''
    Returns a string containing all variables
    '''
    def __str__(self):
        return "Current timestep: {}.\nStepsize: {}.\nx: {}.\ny: {}.".format(
            self._timestep,
            self._stepsize,
            self._x_values,
            self._y_values
        )

'''
Tests
'''
if __name__ == '__main__':
    import matplotlib.pyplot as plt

    euler1 = Euler(lambda x, y: 1, 0, 0)
    euler2 = Euler(lambda x, y: y, 0, 0)
    euler3 = Euler(lambda x, y: y, 0, 1)
    euler4 = Euler(lambda x, y: y * y, -1, 1)
    euler5 = Euler(lambda x, y: y * y, 1, 1)

    euler1.generateN(10)
    euler2.generateN(10)
    euler3.generateN(5)
    euler4.generateN(1)
    euler5.generateN(10)

    print euler1, euler2, euler3, euler4, euler5

    # ax = plt.gca()
    # ax.set_yscale("log")
    plt.subplot(231)
    plt.plot(euler1.getXValues(), euler1.getYValues())
    plt.subplot(232)
    plt.plot(euler2.getXValues(), euler2.getYValues())
    plt.subplot(233)
    plt.plot(euler3.getXValues(), euler3.getYValues())
    plt.subplot(234)
    plt.plot(euler4.getXValues(), euler4.getYValues())
    plt.subplot(235)
    plt.plot(euler5.getXValues(), euler5.getYValues())
    plt.show()
<|endoftext|>


<commit_before>"""Exception handling.
"""

import sys
import warnings
try:
    # Python 3.3 and newer
    from collections.abc import Mapping
except ImportError:
    # Python 2
    from collections import Mapping
import suds

__all__ = [
    # Exceptions thrown by the ICAT or IDS server
    'ServerError',
    'ICATError', 'ICATParameterError', 'ICATInternalError',
    'ICATPrivilegesError', 'ICATNoObjectError', 'ICATObjectExistsError',
    'ICATSessionError', 'ICATValidationError', 'ICATNotImplementedError',
    'IDSError', 'IDSBadRequestError', 'IDSDataNotOnlineError',
    'IDSInsufficientPrivilegesError', 'IDSInsufficientStorageError',
    'IDSInternalError', 'IDSNotFoundError', 'IDSNotImplementedError',
    'translateError',
    # Internal error
    'InternalError',
    # icat.config
    'ConfigError',
    # icat.query
    'QueryNullableOrderWarning',
    # icat.client, icat.entity
    'ClientVersionWarning', 'ICATDeprecationWarning',
    'EntityTypeError', 'VersionMethodError', 'SearchResultError',
    'SearchAssertionError', 'DataConsistencyError',
    # icat.ids
    'IDSResponseError',
    # icat.icatcheck
    'GenealogyError',
    ]

# =========================== base class ===========================

class _BaseException(Exception):
    """An exception that tries to suppress misleading context.

    `Exception Chaining and Embedded Tracebacks`_ has been introduced
    with Python 3.  Unfortunately the result is completely misleading
    most of the times.  This class tries to strip the context from the
    exception traceback.  This works with Python 3.3 and newer, but
    has no effect with Python 3.1 and 3.2.

    This is the common base class for for all exceptions defined in
    :mod:`icat.exception`, it is not intented to be raised directly.

    .. _Exception Chaining and Embedded Tracebacks: https://www.python.org/dev/peps/pep-3134

    """
    def __init__(self, *args):
        super(_BaseException, self).__init__(*args)
        if hasattr(self, '__cause__'):
            self.__cause__ = None

def stripCause(e):
    """Try to suppress misleading context from an exception.

    .. deprecated:: 0.14.0
       Not needed any more, embedded in
       :exc:`icat.exception._BaseException` now.
    """
    warnings.warn("stripCause() is deprecated.", DeprecationWarning, 2)
    if hasattr(e, '__cause__'):
        e.__cause__ = None
    return e

# ========== Exceptions thrown by the ICAT or IDS server ===========

class ServerError(_BaseException):
    """Errors raised by either the ICAT or the IDS server.

    This is the common base class for :exc:`icat.exception.ICATError`
    and :exc:`icat.exception.IDSError`, it is not intented to be
    raised directly.
    """
    def __init__(self, error, status=None):
        """Expecept either a suds.WebFault or a Mapping with the keys 'code',
        'message', and 'offset'.
        """
        if isinstance(error, suds.WebFault):
            try:
                message = self._convertmsg(error.fault.faultstring)
            except AttributeError:
                message = self._convertmsg(str(error))
            super(ServerError, self).__init__(message)
            self.status = status
            self.message = message
            self.fault = error.fault
            self.document = error.document
            try:
                icatexception = error.fault.detail.IcatException
            except AttributeError:
                self.type = None
                self.offset = None
            else:
                self.type = getattr(icatexception, 'type', None)
                self.offset = getattr(icatexception, 'offset', None)
        elif isinstance(error, Mapping):
            # Deliberately not fetching KeyError here.  Require the
            # field to be present.  Only 'offset' is optional.
            message = self._convertmsg(error['message'])
            super(ServerError, self).__init__(message)
            self.status = status
            self.message = message
            self.type = str(error['code'])
            self.offset = error.get('offset', None)
        elif isinstance(error, basestring):
            # For compatibility with other exception classes, also
            # allow the constructor to be called with just a message
            # string as argument.
            message = self._convertmsg(error)
            super(ServerError, self).__init__(message)
            self.status = None
            self.message = message
            self.type = None
            self.offset = None
        else:
            raise TypeError("Invalid argument type '%s'." % type(error))

        # Sanitize offset: we get is as a string from the HTTP
        # response, but it is supposed to be an int.  If the offset is
        # negativ, it has no meaning and should be set to None.
        if self.offset is not None:
            self.offset = int(self.offset)
            if self.offset < 0:
                self.offset = None

    def _convertmsg(self, msg):
        # msg may be a str, a suds.sax.text.Text instance, or (only
        # for Python 2) an unicode instance.  In Python 2, we must
        # convert it to a pure ascii string for Exception.  In Python
        # 3, we convert it to a str.  It may still contain non-ascii
        # chars, but this is ok for Exception in Python 3.
        if sys.version_info < (3, 0):
            return msg.encode('ascii', 'replace')
        else:
            return str(msg)

class ICATError(ServerError):
    """Base class for the errors raised by the ICAT server.
    """
    pass

class ICATParameterError(ICATError):
    """Generally indicates a problem with the arguments made to a
    call.
    """
    pass

class ICATInternalError(ICATError):
    """May be caused by network problems, database problems, GlassFish
    problems or bugs in ICAT.
    """
    pass

class ICATPrivilegesError(ICATError):
    """Indicates that the authorization rules have not matched your
    request.
    """
    pass

class ICATNoObjectError(ICATError):
    """Is thrown when something is not found.
    """
    pass

class ICATObjectExistsError(ICATError):
    """Is thrown when trying to create something but there is already
    one with the same values of the constraint fields.
    """
    pass

class ICATSessionError(ICATError):
    """Is used when the sessionId you have passed into a call is not
    valid or if you are unable to authenticate.
    """
    pass

class ICATValidationError(ICATError):
    """Marks an exception which was thrown instead of placing the
    database in an invalid state.
    """
    pass

class ICATNotImplementedError(ICATError):
    """
    """
    # Added in icat.server 4.6, but not documented.
    pass

IcatExceptionTypeMap = {
    "BAD_PARAMETER": ICATParameterError,
    "INTERNAL": ICATInternalError,
    "INSUFFICIENT_PRIVILEGES": ICATPrivilegesError,
    "NO_SUCH_OBJECT_FOUND": ICATNoObjectError,
    "OBJECT_ALREADY_EXISTS": ICATObjectExistsError,
    "SESSION": ICATSessionError,
    "VALIDATION": ICATValidationError,
    "NOT_IMPLEMENTED": ICATNotImplementedError,
}
"""Map exception types thrown by the ICAT server to Python classes."""

class IDSError(ServerError):
    """Base class for the errors raised by the IDS server.
    """
    pass

class IDSBadRequestError(IDSError):
    """Any kind of bad input parameter.
    """
    pass

class IDSDataNotOnlineError(IDSError):
    """The requested data are not on line.
    """
    pass

class IDSInsufficientPrivilegesError(IDSError):
    """You are denied access to the data.
    """
    pass

class IDSInsufficientStorageError(IDSError):
    """There is not sufficient physical storage or you have exceeded some quota.
    """
    pass

class IDSInternalError(IDSError):
    """Some kind of failure in the server or in communicating with the server.
    """
    pass

class IDSNotFoundError(IDSError):
    """The requested data do not exist.
    """
    pass

class IDSNotImplementedError(IDSError):
    """Use of some functionality that is not supported by the implementation.
    """
    pass

IDSExceptionTypeMap = {
    "BadRequestException": IDSBadRequestError,
    "DataNotOnlineException": IDSDataNotOnlineError,
    "InsufficientPrivilegesException": IDSInsufficientPrivilegesError,
    "InsufficientStorageException": IDSInsufficientStorageError,
    "InternalException": IDSInternalError,
    "NotFoundException": IDSNotFoundError,
    "NotImplementedException": IDSNotImplementedError,
}
"""Map IDS error codes to Python classes."""

def translateError(error, status=None, server="ICAT"):
    """Translate an error from ICAT or IDS to the corresponding exception."""
    if server == "ICAT":
        typemap = IcatExceptionTypeMap
        BaseClass = ICATError
    elif server == "IDS":
        typemap = IDSExceptionTypeMap
        BaseClass = IDSError
    else:
        raise ValueError("Invalid server '%s'." % server)

    if isinstance(error, suds.WebFault):
        try:
            Class = typemap[error.fault.detail.IcatException.type]
        except AttributeError:
            Class = BaseClass
    elif isinstance(error, Mapping):
        Class = typemap[error['code']]
    else:
        raise TypeError("Invalid argument type '%s'." % type(error))

    return Class(error, status)

# ========================= Internal error =========================

class InternalError(_BaseException):
    """An error that reveals a bug in python-icat.
    """
    pass

# ================ Exceptions raised in icat.config ================

class ConfigError(_BaseException):
    """Error getting configuration options."""
    pass

# ================ Exceptions raised in icat.query =================

class QueryNullableOrderWarning(Warning):
    """Warn about using a nullable relation for ordering.
    """
    def __init__(self, attr):
        msg = ("ordering on a nullable relation implicitly "
               "adds a '%s IS NOT NULL' condition." % attr)
        super(QueryNullableOrderWarning, self).__init__(msg)

# ======== Exceptions raised in icat.client and icat.entity ========

class ClientVersionWarning(Warning):
    """Warn that the version of the ICAT server is not supported by
    the client.
    """
    def __init__(self, version=None, comment=None):
        if version is None:
            icatstr = "this ICAT version"
        else:
            icatstr = "ICAT version %s" % version
        if comment is None:
            msg = ("%s is not supported, "
                   "expect problems and weird behavior!" % icatstr)
        else:
            msg = ("%s is not supported (%s), "
                   "expect problems and weird behavior!" % (icatstr, comment))
        super(ClientVersionWarning, self).__init__(msg)

class ICATDeprecationWarning(DeprecationWarning):
    """Warn about using an API feature that may get removed in future ICAT
    server versions.
    """
    def __init__(self, feature, version=None):
        if version is None:
            icatstr = "a future ICAT version"
        else:
            icatstr = "ICAT version %s" % version
        msg = ("%s has been deprecated and is expected to get removed in %s."
               % (feature, icatstr))
        super(ICATDeprecationWarning, self).__init__(msg)

class EntityTypeError(_BaseException):
    """An invalid entity type has been used."""
    pass

class VersionMethodError(_BaseException):
    """Call of an API method that is not supported in the version
    of the server.
    """
    def __init__(self, method, version=None, service="ICAT"):
        if version is None:
            icatstr = "this %s version" % service
        else:
            icatstr = "%s version %s" % (service, version)
        msg = ("%s is not supported in %s." % (method, icatstr))
        super(VersionMethodError, self).__init__(msg)

class SearchResultError(_BaseException):
    """A search result does not conform to what should have been expected.
    """
    pass

class SearchAssertionError(SearchResultError):
    """A search result does not conform to an assertion.

    This exception is thrown when the number of objects found on a
    search does not lie within the bounds of an assertion, see
    :meth:`icat.client.Client.assertedSearch`.
    """
    def __init__(self, query, assertmin, assertmax, num):
        # The most common case will be assertmin > 0 and num = 0.
        # Formulate a convenient message for this case and a generic
        # one for all other cases.
        if num == 0:
            msg = 'Nothing found on query: "%s"' % query
        elif num < assertmin:
            msg = ('Less objects found then expected (%d < %d) on query: "%s"'
                   % (num, assertmin, query))
        elif assertmax == 1:
            msg = ('Search result is not unique, '
                   '%d objects found on query: "%s"'
                   % (num, query))
        else:
            msg = ('Number of objects found (%d) is not within '
                   'the expected bounds between %d and %d on query: "%s"'
                   % (num, assertmin, assertmax, query))
        super(SearchAssertionError, self).__init__(msg)
        self.query = query
        self.assertmin = assertmin
        self.assertmax = assertmax
        self.num = num

class DataConsistencyError(_BaseException):
    """Some data is not consistent with rules or constraints."""
    pass

# ================= Exceptions raised in icat.ids ==================

class IDSResponseError(_BaseException):
    """The response from the IDS was not what should have been expected.
    """
    pass

# ============== Exceptions raised in icat.icatcheck ===============

class GenealogyError(_BaseException):
    """Error in the genealogy of entity types.

    .. deprecated:: 0.17
       Only used in :mod:`icat.icatcheck` which in turn is deprecated.
    """
    pass

<commit_msg>Minor documentation fix: old Python versions no longer supported anyway.<commit_after>"""Exception handling.
"""

import sys
import warnings
try:
    # Python 3.3 and newer
    from collections.abc import Mapping
except ImportError:
    # Python 2
    from collections import Mapping
import suds

__all__ = [
    # Exceptions thrown by the ICAT or IDS server
    'ServerError',
    'ICATError', 'ICATParameterError', 'ICATInternalError',
    'ICATPrivilegesError', 'ICATNoObjectError', 'ICATObjectExistsError',
    'ICATSessionError', 'ICATValidationError', 'ICATNotImplementedError',
    'IDSError', 'IDSBadRequestError', 'IDSDataNotOnlineError',
    'IDSInsufficientPrivilegesError', 'IDSInsufficientStorageError',
    'IDSInternalError', 'IDSNotFoundError', 'IDSNotImplementedError',
    'translateError',
    # Internal error
    'InternalError',
    # icat.config
    'ConfigError',
    # icat.query
    'QueryNullableOrderWarning',
    # icat.client, icat.entity
    'ClientVersionWarning', 'ICATDeprecationWarning',
    'EntityTypeError', 'VersionMethodError', 'SearchResultError',
    'SearchAssertionError', 'DataConsistencyError',
    # icat.ids
    'IDSResponseError',
    # icat.icatcheck
    'GenealogyError',
    ]

# =========================== base class ===========================

class _BaseException(Exception):
    """An exception that tries to suppress misleading context.

    `Exception Chaining and Embedded Tracebacks`_ has been introduced
    with Python 3.  Unfortunately the result is completely misleading
    most of the times.  This class tries to strip the context from the
    exception traceback.

    This is the common base class for for all exceptions defined in
    :mod:`icat.exception`, it is not intented to be raised directly.

    .. _Exception Chaining and Embedded Tracebacks: https://www.python.org/dev/peps/pep-3134

    """
    def __init__(self, *args):
        super(_BaseException, self).__init__(*args)
        if hasattr(self, '__cause__'):
            self.__cause__ = None

def stripCause(e):
    """Try to suppress misleading context from an exception.

    .. deprecated:: 0.14.0
       Not needed any more, embedded in
       :exc:`icat.exception._BaseException` now.
    """
    warnings.warn("stripCause() is deprecated.", DeprecationWarning, 2)
    if hasattr(e, '__cause__'):
        e.__cause__ = None
    return e

# ========== Exceptions thrown by the ICAT or IDS server ===========

class ServerError(_BaseException):
    """Errors raised by either the ICAT or the IDS server.

    This is the common base class for :exc:`icat.exception.ICATError`
    and :exc:`icat.exception.IDSError`, it is not intented to be
    raised directly.
    """
    def __init__(self, error, status=None):
        """Expecept either a suds.WebFault or a Mapping with the keys 'code',
        'message', and 'offset'.
        """
        if isinstance(error, suds.WebFault):
            try:
                message = self._convertmsg(error.fault.faultstring)
            except AttributeError:
                message = self._convertmsg(str(error))
            super(ServerError, self).__init__(message)
            self.status = status
            self.message = message
            self.fault = error.fault
            self.document = error.document
            try:
                icatexception = error.fault.detail.IcatException
            except AttributeError:
                self.type = None
                self.offset = None
            else:
                self.type = getattr(icatexception, 'type', None)
                self.offset = getattr(icatexception, 'offset', None)
        elif isinstance(error, Mapping):
            # Deliberately not fetching KeyError here.  Require the
            # field to be present.  Only 'offset' is optional.
            message = self._convertmsg(error['message'])
            super(ServerError, self).__init__(message)
            self.status = status
            self.message = message
            self.type = str(error['code'])
            self.offset = error.get('offset', None)
        elif isinstance(error, basestring):
            # For compatibility with other exception classes, also
            # allow the constructor to be called with just a message
            # string as argument.
            message = self._convertmsg(error)
            super(ServerError, self).__init__(message)
            self.status = None
            self.message = message
            self.type = None
            self.offset = None
        else:
            raise TypeError("Invalid argument type '%s'." % type(error))

        # Sanitize offset: we get is as a string from the HTTP
        # response, but it is supposed to be an int.  If the offset is
        # negativ, it has no meaning and should be set to None.
        if self.offset is not None:
            self.offset = int(self.offset)
            if self.offset < 0:
                self.offset = None

    def _convertmsg(self, msg):
        # msg may be a str, a suds.sax.text.Text instance, or (only
        # for Python 2) an unicode instance.  In Python 2, we must
        # convert it to a pure ascii string for Exception.  In Python
        # 3, we convert it to a str.  It may still contain non-ascii
        # chars, but this is ok for Exception in Python 3.
        if sys.version_info < (3, 0):
            return msg.encode('ascii', 'replace')
        else:
            return str(msg)

class ICATError(ServerError):
    """Base class for the errors raised by the ICAT server.
    """
    pass

class ICATParameterError(ICATError):
    """Generally indicates a problem with the arguments made to a
    call.
    """
    pass

class ICATInternalError(ICATError):
    """May be caused by network problems, database problems, GlassFish
    problems or bugs in ICAT.
    """
    pass

class ICATPrivilegesError(ICATError):
    """Indicates that the authorization rules have not matched your
    request.
    """
    pass

class ICATNoObjectError(ICATError):
    """Is thrown when something is not found.
    """
    pass

class ICATObjectExistsError(ICATError):
    """Is thrown when trying to create something but there is already
    one with the same values of the constraint fields.
    """
    pass

class ICATSessionError(ICATError):
    """Is used when the sessionId you have passed into a call is not
    valid or if you are unable to authenticate.
    """
    pass

class ICATValidationError(ICATError):
    """Marks an exception which was thrown instead of placing the
    database in an invalid state.
    """
    pass

class ICATNotImplementedError(ICATError):
    """
    """
    # Added in icat.server 4.6, but not documented.
    pass

IcatExceptionTypeMap = {
    "BAD_PARAMETER": ICATParameterError,
    "INTERNAL": ICATInternalError,
    "INSUFFICIENT_PRIVILEGES": ICATPrivilegesError,
    "NO_SUCH_OBJECT_FOUND": ICATNoObjectError,
    "OBJECT_ALREADY_EXISTS": ICATObjectExistsError,
    "SESSION": ICATSessionError,
    "VALIDATION": ICATValidationError,
    "NOT_IMPLEMENTED": ICATNotImplementedError,
}
"""Map exception types thrown by the ICAT server to Python classes."""

class IDSError(ServerError):
    """Base class for the errors raised by the IDS server.
    """
    pass

class IDSBadRequestError(IDSError):
    """Any kind of bad input parameter.
    """
    pass

class IDSDataNotOnlineError(IDSError):
    """The requested data are not on line.
    """
    pass

class IDSInsufficientPrivilegesError(IDSError):
    """You are denied access to the data.
    """
    pass

class IDSInsufficientStorageError(IDSError):
    """There is not sufficient physical storage or you have exceeded some quota.
    """
    pass

class IDSInternalError(IDSError):
    """Some kind of failure in the server or in communicating with the server.
    """
    pass

class IDSNotFoundError(IDSError):
    """The requested data do not exist.
    """
    pass

class IDSNotImplementedError(IDSError):
    """Use of some functionality that is not supported by the implementation.
    """
    pass

IDSExceptionTypeMap = {
    "BadRequestException": IDSBadRequestError,
    "DataNotOnlineException": IDSDataNotOnlineError,
    "InsufficientPrivilegesException": IDSInsufficientPrivilegesError,
    "InsufficientStorageException": IDSInsufficientStorageError,
    "InternalException": IDSInternalError,
    "NotFoundException": IDSNotFoundError,
    "NotImplementedException": IDSNotImplementedError,
}
"""Map IDS error codes to Python classes."""

def translateError(error, status=None, server="ICAT"):
    """Translate an error from ICAT or IDS to the corresponding exception."""
    if server == "ICAT":
        typemap = IcatExceptionTypeMap
        BaseClass = ICATError
    elif server == "IDS":
        typemap = IDSExceptionTypeMap
        BaseClass = IDSError
    else:
        raise ValueError("Invalid server '%s'." % server)

    if isinstance(error, suds.WebFault):
        try:
            Class = typemap[error.fault.detail.IcatException.type]
        except AttributeError:
            Class = BaseClass
    elif isinstance(error, Mapping):
        Class = typemap[error['code']]
    else:
        raise TypeError("Invalid argument type '%s'." % type(error))

    return Class(error, status)

# ========================= Internal error =========================

class InternalError(_BaseException):
    """An error that reveals a bug in python-icat.
    """
    pass

# ================ Exceptions raised in icat.config ================

class ConfigError(_BaseException):
    """Error getting configuration options."""
    pass

# ================ Exceptions raised in icat.query =================

class QueryNullableOrderWarning(Warning):
    """Warn about using a nullable relation for ordering.
    """
    def __init__(self, attr):
        msg = ("ordering on a nullable relation implicitly "
               "adds a '%s IS NOT NULL' condition." % attr)
        super(QueryNullableOrderWarning, self).__init__(msg)

# ======== Exceptions raised in icat.client and icat.entity ========

class ClientVersionWarning(Warning):
    """Warn that the version of the ICAT server is not supported by
    the client.
    """
    def __init__(self, version=None, comment=None):
        if version is None:
            icatstr = "this ICAT version"
        else:
            icatstr = "ICAT version %s" % version
        if comment is None:
            msg = ("%s is not supported, "
                   "expect problems and weird behavior!" % icatstr)
        else:
            msg = ("%s is not supported (%s), "
                   "expect problems and weird behavior!" % (icatstr, comment))
        super(ClientVersionWarning, self).__init__(msg)

class ICATDeprecationWarning(DeprecationWarning):
    """Warn about using an API feature that may get removed in future ICAT
    server versions.
    """
    def __init__(self, feature, version=None):
        if version is None:
            icatstr = "a future ICAT version"
        else:
            icatstr = "ICAT version %s" % version
        msg = ("%s has been deprecated and is expected to get removed in %s."
               % (feature, icatstr))
        super(ICATDeprecationWarning, self).__init__(msg)

class EntityTypeError(_BaseException):
    """An invalid entity type has been used."""
    pass

class VersionMethodError(_BaseException):
    """Call of an API method that is not supported in the version
    of the server.
    """
    def __init__(self, method, version=None, service="ICAT"):
        if version is None:
            icatstr = "this %s version" % service
        else:
            icatstr = "%s version %s" % (service, version)
        msg = ("%s is not supported in %s." % (method, icatstr))
        super(VersionMethodError, self).__init__(msg)

class SearchResultError(_BaseException):
    """A search result does not conform to what should have been expected.
    """
    pass

class SearchAssertionError(SearchResultError):
    """A search result does not conform to an assertion.

    This exception is thrown when the number of objects found on a
    search does not lie within the bounds of an assertion, see
    :meth:`icat.client.Client.assertedSearch`.
    """
    def __init__(self, query, assertmin, assertmax, num):
        # The most common case will be assertmin > 0 and num = 0.
        # Formulate a convenient message for this case and a generic
        # one for all other cases.
        if num == 0:
            msg = 'Nothing found on query: "%s"' % query
        elif num < assertmin:
            msg = ('Less objects found then expected (%d < %d) on query: "%s"'
                   % (num, assertmin, query))
        elif assertmax == 1:
            msg = ('Search result is not unique, '
                   '%d objects found on query: "%s"'
                   % (num, query))
        else:
            msg = ('Number of objects found (%d) is not within '
                   'the expected bounds between %d and %d on query: "%s"'
                   % (num, assertmin, assertmax, query))
        super(SearchAssertionError, self).__init__(msg)
        self.query = query
        self.assertmin = assertmin
        self.assertmax = assertmax
        self.num = num

class DataConsistencyError(_BaseException):
    """Some data is not consistent with rules or constraints."""
    pass

# ================= Exceptions raised in icat.ids ==================

class IDSResponseError(_BaseException):
    """The response from the IDS was not what should have been expected.
    """
    pass

# ============== Exceptions raised in icat.icatcheck ===============

class GenealogyError(_BaseException):
    """Error in the genealogy of entity types.

    .. deprecated:: 0.17
       Only used in :mod:`icat.icatcheck` which in turn is deprecated.
    """
    pass

<|endoftext|>