<commit_before>#Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import time
import argparse
import logging
import paddle.fluid as fluid
import paddle
import utils
import numpy as np
from nets import SequenceSemanticRetrieval

logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def parse_args():
    parser = argparse.ArgumentParser("sequence semantic retrieval")
    parser.add_argument(
        "--train_dir", type=str, default='train_data', help="Training file")
    parser.add_argument(
        "--base_lr", type=float, default=0.01, help="learning rate")
    parser.add_argument(
        '--vocab_path', type=str, default='vocab.txt', help='vocab file')
    parser.add_argument(
        "--epochs", type=int, default=10, help="Number of epochs")
    parser.add_argument(
        '--parallel', type=int, default=0, help='whether parallel')
    parser.add_argument(
        '--use_cuda', type=int, default=0, help='whether use gpu')
    parser.add_argument(
        '--print_batch', type=int, default=10, help='num of print batch')
    parser.add_argument(
        '--model_dir', type=str, default='model_output', help='model dir')
    parser.add_argument(
        "--hidden_size", type=int, default=128, help="hidden size")
    parser.add_argument(
        "--batch_size", type=int, default=50, help="number of batch")
    parser.add_argument(
        "--embedding_dim", type=int, default=128, help="embedding dim")
    parser.add_argument(
        '--num_devices', type=int, default=1, help='Number of GPU devices')
    parser.add_argument(
        '--step_num', type=int, default=1000, help='Number of steps')
    parser.add_argument(
        '--enable_ce',
        action='store_true',
        help='If set, run the task with continuous evaluation logs.')
    return parser.parse_args()

def get_cards(args):
    return args.num_devices

def train(args):
    if args.enable_ce:
        SEED = 102
        fluid.default_startup_program().random_seed = SEED
        fluid.default_main_program().random_seed = SEED
    use_cuda = True if args.use_cuda else False
    parallel = True if args.parallel else False
    print("use_cuda:", use_cuda, "parallel:", parallel)
    train_reader, vocab_size = utils.construct_train_data(
        args.train_dir, args.vocab_path, args.batch_size * get_cards(args))
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
    ssr = SequenceSemanticRetrieval(vocab_size, args.embedding_dim,
                                    args.hidden_size)
    # Train program
    train_input_data, cos_pos, avg_cost, acc = ssr.train()

    # Optimization to minimize lost
    optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr)
    optimizer.minimize(avg_cost)

    data_list = [var.name for var in train_input_data]
    print(data_list)
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())
    loader = fluid.io.DataLoader.from_generator(
        feed_list=train_input_data, capacity=10000, iterable=True)
    loader.set_sample_list_generator(train_reader, places=place)
    if parallel:
        train_exe = fluid.ParallelExecutor(
            use_cuda=use_cuda, loss_name=avg_cost.name)
    else:
        train_exe = exe

    total_time = 0.0
    ce_info = []
    for pass_id in range(args.epochs):
        epoch_idx = pass_id + 1
        print("epoch_%d start" % epoch_idx)
        t0 = time.time()
        i = 0
        for batch_id, data in enumerate(loader()):
            i += 1
            loss_val, correct_val = train_exe.run(
                feed=data, fetch_list=[avg_cost.name, acc.name])
            ce_info.append(float(np.mean(correct_val)) / args.batch_size)
            if i % args.print_batch == 0:
                logger.info(
                    "Train --> pass: {} batch_id: {} avg_cost: {}, acc: {}".
                    format(pass_id, batch_id,
                           np.mean(loss_val),
                           float(np.mean(correct_val)) / args.batch_size))
            if args.enable_ce and i > args.step_num:
                break
        t1 = time.time()
        total_time += t1 - t0
        print("epoch:%d num_steps:%d time_cost(s):%f" %
              (epoch_idx, i, total_time / epoch_idx))
        save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx)
        fluid.save(fluid.default_main_program(), model_path=save_dir)
        print("model saved in %s" % save_dir)

    # only for ce
    if args.enable_ce:
        ce_acc = 0
        try:
            ce_acc = ce_info[-2]
        except:
            print("ce info error")
        epoch_idx = args.epochs
        device = get_device(args)
        if args.use_cuda:
            gpu_num = device[1]
            print("kpis\teach_pass_duration_gpu%s\t%s" %
                  (gpu_num, total_time / epoch_idx))
            print("kpis\ttrain_acc_gpu%s\t%s" % (gpu_num, ce_acc))
        else:
            cpu_num = device[1]
            threads_num = device[2]
            print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" %
                  (cpu_num, threads_num, total_time / epoch_idx))
            print("kpis\ttrain_acc_cpu%s_thread%s\t%s" %
                  (cpu_num, threads_num, ce_acc))

def get_device(args):
    if args.use_cuda:
        gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1)
        gpu_num = len(gpus.split(','))
        return "gpu", gpu_num
    else:
        threads_num = os.environ.get('NUM_THREADS', 1)
        cpu_num = os.environ.get('CPU_NUM', 1)
        return "cpu", int(cpu_num), int(threads_num)

def main():
    args = parse_args()
    train(args)

if __name__ == "__main__":
    utils.check_version()
    main()
<commit_msg>fix ssr (#4370)<commit_after>#Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import time
import argparse
import logging
import paddle.fluid as fluid
import paddle
import utils
import numpy as np
from nets import SequenceSemanticRetrieval

logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def parse_args():
    parser = argparse.ArgumentParser("sequence semantic retrieval")
    parser.add_argument(
        "--train_dir", type=str, default='train_data', help="Training file")
    parser.add_argument(
        "--base_lr", type=float, default=0.01, help="learning rate")
    parser.add_argument(
        '--vocab_path', type=str, default='vocab.txt', help='vocab file')
    parser.add_argument(
        "--epochs", type=int, default=10, help="Number of epochs")
    parser.add_argument(
        '--parallel', type=int, default=0, help='whether parallel')
    parser.add_argument(
        '--use_cuda', type=int, default=0, help='whether use gpu')
    parser.add_argument(
        '--print_batch', type=int, default=10, help='num of print batch')
    parser.add_argument(
        '--model_dir', type=str, default='model_output', help='model dir')
    parser.add_argument(
        "--hidden_size", type=int, default=128, help="hidden size")
    parser.add_argument(
        "--batch_size", type=int, default=50, help="number of batch")
    parser.add_argument(
        "--embedding_dim", type=int, default=128, help="embedding dim")
    parser.add_argument(
        '--num_devices', type=int, default=1, help='Number of GPU devices')
    parser.add_argument(
        '--step_num', type=int, default=1000, help='Number of steps')
    parser.add_argument(
        '--enable_ce',
        action='store_true',
        help='If set, run the task with continuous evaluation logs.')
    return parser.parse_args()

def get_cards(args):
    return args.num_devices

def train(args):
    if args.enable_ce:
        SEED = 102
        fluid.default_startup_program().random_seed = SEED
        fluid.default_main_program().random_seed = SEED
    use_cuda = True if args.use_cuda else False
    parallel = True if args.parallel else False
    print("use_cuda:", use_cuda, "parallel:", parallel)
    train_reader, vocab_size = utils.construct_train_data(
        args.train_dir, args.vocab_path, args.batch_size * get_cards(args))
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
    ssr = SequenceSemanticRetrieval(vocab_size, args.embedding_dim,
                                    args.hidden_size)
    # Train program
    train_input_data, cos_pos, avg_cost, acc = ssr.train()

    # Optimization to minimize lost
    optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr)
    optimizer.minimize(avg_cost)

    data_list = [var.name for var in train_input_data]
    print(data_list)
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())
    loader = fluid.io.DataLoader.from_generator(
        feed_list=train_input_data, capacity=10000, iterable=True)
    loader.set_sample_list_generator(train_reader, places=place)
    if parallel:
        train_exe = exe
        #train_exe = fluid.ParallelExecutor(
        #    use_cuda=use_cuda, loss_name=avg_cost.name)
    else:
        train_exe = exe

    total_time = 0.0
    ce_info = []
    for pass_id in range(args.epochs):
        epoch_idx = pass_id + 1
        print("epoch_%d start" % epoch_idx)
        t0 = time.time()
        i = 0
        for batch_id, data in enumerate(loader()):
            i += 1
            loss_val, correct_val = train_exe.run(
                feed=data, fetch_list=[avg_cost.name, acc.name])
            ce_info.append(float(np.mean(correct_val)) / args.batch_size)
            if i % args.print_batch == 0:
                logger.info(
                    "Train --> pass: {} batch_id: {} avg_cost: {}, acc: {}".
                    format(pass_id, batch_id,
                           np.mean(loss_val),
                           float(np.mean(correct_val)) / args.batch_size))
            if args.enable_ce and i > args.step_num:
                break
        t1 = time.time()
        total_time += t1 - t0
        print("epoch:%d num_steps:%d time_cost(s):%f" %
              (epoch_idx, i, total_time / epoch_idx))
        save_dir = "%s/epoch_%d" % (args.model_dir, epoch_idx)
        fluid.save(fluid.default_main_program(), model_path=save_dir)
        print("model saved in %s" % save_dir)

    # only for ce
    if args.enable_ce:
        ce_acc = 0
        try:
            ce_acc = ce_info[-2]
        except:
            print("ce info error")
        epoch_idx = args.epochs
        device = get_device(args)
        if args.use_cuda:
            gpu_num = device[1]
            print("kpis\teach_pass_duration_gpu%s\t%s" %
                  (gpu_num, total_time / epoch_idx))
            print("kpis\ttrain_acc_gpu%s\t%s" % (gpu_num, ce_acc))
        else:
            cpu_num = device[1]
            threads_num = device[2]
            print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" %
                  (cpu_num, threads_num, total_time / epoch_idx))
            print("kpis\ttrain_acc_cpu%s_thread%s\t%s" %
                  (cpu_num, threads_num, ce_acc))

def get_device(args):
    if args.use_cuda:
        gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1)
        gpu_num = len(gpus.split(','))
        return "gpu", gpu_num
    else:
        threads_num = os.environ.get('NUM_THREADS', 1)
        cpu_num = os.environ.get('CPU_NUM', 1)
        return "cpu", int(cpu_num), int(threads_num)

def main():
    args = parse_args()
    train(args)

if __name__ == "__main__":
    utils.check_version()
    main()
<|endoftext|>


<commit_before>'''
Routines to set up a minion
'''
# Import python libs
import os
import sys
import imp
import distutils.sysconfig

# Import cython
import pyximport; pyximport.install()

def minion_mods(opts):
    '''
    Returns the minion modules
    '''
    module_dirs = [
        os.path.join(distutils.sysconfig.get_python_lib(), 'salt/modules'),
        ] + opts['module_dirs']
    load = Loader(module_dirs, opts)
    return load.apply_introspection(load.gen_functions())

def returners(opts):
    '''
    Returns the returner modules
    '''
    module_dirs = [
        os.path.join(distutils.sysconfig.get_python_lib(), 'salt/returners'),
        ] + opts['returner_dirs']
    load = Loader(module_dirs, opts)
    return load.filter_func('returner')

class Loader(object):
    '''
    Used to load in arbitrairy modules from a directory, the Loader can also be
    used to only load specific functions from a directory, or to call modules
    in an arbitrairy directory directly.
    '''
    def __init__(self, module_dirs, opts={}):
        self.module_dirs = module_dirs
        self.opts = self.__prep_mod_opts(opts)

    def __prep_mod_opts(self, opts):
        '''
        Strip out of the opts any logger instance
        '''
        mod_opts = {}
        for key, val in opts.items():
            if key == 'logger':
                continue
            mod_opts[key] = val
        return mod_opts

    def get_docs(self, funcs, module=''):
        '''
        Return a dict containing all of the doc strings in the functions dict
        '''
        docs = {}
        for fun in self.functions:
            if fun.startswith(module):
                docs[fun] = self.functions[fun].__doc__
        return docs

    def call(self, fun, arg=[]):
        '''
        Call a function in the load path.
        '''
        name = fun[:fun.rindex('.')]
        fn_, path, desc = imp.find_module(name, self.module_dirs)
        mod = imp.load_module(name, fn_, path, desc)
        return getattr(mod, fun[fun.rindex('.'):])(*arg)

    def gen_functions(self):
        '''
        Return a dict of functions found in the defined module_dirs
        '''
        mods = set()
        funcs = {}
        for mod_dir in self.module_dirs:
            for fn_ in os.listdir(mod_dir):
                if fn_.startswith('_'):
                    continue
                if fn_.endswith('.py')\
                    or fn_.endswith('.pyc')\
                    or fn_.endswith('.pyo')\
                    or fn_.endswith('.so')\
                    or fn_.endswith('.pyx'):
                    mods.add(fn_[:fn_.rindex('.')])
        for name in mods:
            try:
                fn_, path, desc = imp.find_module(name, self.module_dirs)
                mod = imp.load_module(name, fn_, path, desc)
            except ImportError:
                continue
            if hasattr(mod, '__opts__'):
                mod.__opts__.update(self.opts)
            else:
                mod.__opts__ = self.opts

            for attr in dir(mod):
                if attr.startswith('_'):
                    continue
                if callable(getattr(mod, attr)):
                    funcs[name + '.' + attr] = getattr(mod, attr)
        return funcs

    def apply_introspection(self, funcs):
        '''
        Pass in a function object returned from get_functions to load in
        introspection functions.
        '''
        funcs['sys.list_functions'] = lambda: self.list_funcs(funcs)
        funcs['sys.list_modules'] = lambda: funcs.keys()
        funcs['sys.doc'] = lambda module='': self.get_docs(funcs, module)
        #funcs['sys.reload_functions'] = self.reload_functions

    def filter_func(self, name):
        '''
        Filter a specific function out of the functions, this is used to load
        the returners for the salt minion
        '''
        funcs = {}
        for key, fun in self.gen_functions().items():
            if key[key.rindex('.'):] == name:
                funcs[key[:key.rindex('.')]] = fun
        return funcs
<commit_msg>Fix docs loading<commit_after>'''
Routines to set up a minion
'''
# Import python libs
import os
import sys
import imp
import distutils.sysconfig

# Import cython
import pyximport; pyximport.install()

def minion_mods(opts):
    '''
    Returns the minion modules
    '''
    module_dirs = [
        os.path.join(distutils.sysconfig.get_python_lib(), 'salt/modules'),
        ] + opts['module_dirs']
    load = Loader(module_dirs, opts)
    return load.apply_introspection(load.gen_functions())

def returners(opts):
    '''
    Returns the returner modules
    '''
    module_dirs = [
        os.path.join(distutils.sysconfig.get_python_lib(), 'salt/returners'),
        ] + opts['returner_dirs']
    load = Loader(module_dirs, opts)
    return load.filter_func('returner')

class Loader(object):
    '''
    Used to load in arbitrairy modules from a directory, the Loader can also be
    used to only load specific functions from a directory, or to call modules
    in an arbitrairy directory directly.
    '''
    def __init__(self, module_dirs, opts={}):
        self.module_dirs = module_dirs
        self.opts = self.__prep_mod_opts(opts)

    def __prep_mod_opts(self, opts):
        '''
        Strip out of the opts any logger instance
        '''
        mod_opts = {}
        for key, val in opts.items():
            if key == 'logger':
                continue
            mod_opts[key] = val
        return mod_opts

    def get_docs(self, funcs, module=''):
        '''
        Return a dict containing all of the doc strings in the functions dict
        '''
        docs = {}
        for fun in funcs:
            if fun.startswith(module):
                docs[fun] = funcs[fun].__doc__
        return docs

    def call(self, fun, arg=[]):
        '''
        Call a function in the load path.
        '''
        name = fun[:fun.rindex('.')]
        fn_, path, desc = imp.find_module(name, self.module_dirs)
        mod = imp.load_module(name, fn_, path, desc)
        return getattr(mod, fun[fun.rindex('.'):])(*arg)

    def gen_functions(self):
        '''
        Return a dict of functions found in the defined module_dirs
        '''
        mods = set()
        funcs = {}
        for mod_dir in self.module_dirs:
            for fn_ in os.listdir(mod_dir):
                if fn_.startswith('_'):
                    continue
                if fn_.endswith('.py')\
                    or fn_.endswith('.pyc')\
                    or fn_.endswith('.pyo')\
                    or fn_.endswith('.so')\
                    or fn_.endswith('.pyx'):
                    mods.add(fn_[:fn_.rindex('.')])
        for name in mods:
            try:
                fn_, path, desc = imp.find_module(name, self.module_dirs)
                mod = imp.load_module(name, fn_, path, desc)
            except ImportError:
                continue
            if hasattr(mod, '__opts__'):
                mod.__opts__.update(self.opts)
            else:
                mod.__opts__ = self.opts

            for attr in dir(mod):
                if attr.startswith('_'):
                    continue
                if callable(getattr(mod, attr)):
                    funcs[name + '.' + attr] = getattr(mod, attr)
        return funcs

    def apply_introspection(self, funcs):
        '''
        Pass in a function object returned from get_functions to load in
        introspection functions.
        '''
        funcs['sys.list_functions'] = lambda: self.list_funcs(funcs)
        funcs['sys.list_modules'] = lambda: funcs.keys()
        funcs['sys.doc'] = lambda module='': self.get_docs(funcs, module)
        #funcs['sys.reload_functions'] = self.reload_functions

    def filter_func(self, name):
        '''
        Filter a specific function out of the functions, this is used to load
        the returners for the salt minion
        '''
        funcs = {}
        for key, fun in self.gen_functions().items():
            if key[key.rindex('.'):] == name:
                funcs[key[:key.rindex('.')]] = fun
        return funcs
<|endoftext|>


<commit_before>from basic import *

# Base class used by all AST nodes deemed to represent a C expression
class Expression(Node):
    pass

# Not entirely sure what this is meant to be?
# It has no children, so either it ignores the type whose size it measures, or
# it does something entirely different.
class SizeOfType(Expression):
    CODE = "241600"
    LABEL = "SizeOfType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], SizeOfType.CODE)
        assert not jsn['children']
        return SizeOfType(jsn['pos'])

class Assignment(Expression):
    CODE = "240700"
    LABEL = "Assignment"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Assignment.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 3
        assert isinstance(children[1], GenericString)
        return Assignment(jsn['pos'], children[0], children[2])

    def __init__(self, pos, lhs, rhs):
        super().__init__(pos)
        self.__lhs = lhs
        self.__rhs = rhs

# This seems to just represent Postfix expressions? This has nothing to do with
# Infix expression?
class Infix(Expression):
    CODE = "240900"
    LABEL = "Infix"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Infix.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 2
        return Infix(jsn['pos'], children[0], children[1])

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class Postfix(Expression):
    CODE = "240800"
    LABEL = "Postfix"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Postfix.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 2
        assert isinstance(children[1], GenericString)
        return Postfix(jsn['pos'], children[0], children[1])

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class RecordPtAccess(Expression):
    CODE = "241400"
    LABEL = "RecordPtAccess"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], RecordPtAccess.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert isinstance(children[1], GenericString)
        return RecordPtAccess(jsn['pos'],\
                              children[0],
                              children[1])

    def __init__(self, pos, expr, member):
        super().__init__(pos)
        self.__expr = expr
        self.__member = member

class Identity(Expression):
    CODE = "240100"
    LABEL = "Ident"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Identity.CODE)
        of = GenericString.from_json(jsn['children'][0])
        return Identity(jsn['pos'], of)

    def __init__(self, pos, of):
        super().__init__(pos)
        self.__of = of

    def of(self):
        return self.__of.read()

    def to_s(self):
        return self.__of.read()

class Constant(Expression):
    CODE = "240200"
    LABEL = "Constant"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Constant.CODE)
        return Constant(jsn['pos'], jsn['label'])

    def __init__(self, pos, value):
        super().__init__(pos)
        self.__value = value

    def value(self):
        return self.__value

    def to_s(self):
        return str(self.__value)

class Cast(Expression):
    CODE = "241700"
    LABEL = "Cast"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Cast.CODE)
        expr = Node.from_json(jsn['children'][0])
        return Cast(jsn['pos'], expr)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def to_s(self):
        return "(UNKNOWN_TYPE) %s" % self.__expr.to_s()

# What is the difference between Init and Assignment?
class InitExpr(Expression):
    CODE = "360100"
    LABEL = "InitExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], InitExpr.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        return InitExpr(jsn['pos'], children)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

class Ternary(Expression):
    CODE = "240500"
    LABEL = "CondExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Ternary.CODE)
        return Ternary(jsn['pos'],\
                       Node.from_json(jsn['children'][0]),\
                       Node.from_json(jsn['children'][1]),\
                       Node.from_json(jsn['children'][2]))

    def __init__(self, pos, cond, then, els):
        super().__init__(pos)
        self.__cond = cond
        self.__then = then
        self.__els = els

class FunctionCall(Expression):
    CODE = "240400"
    LABEL = "FunCall"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], FunctionCall.CODE)
        function = Identity.from_json(jsn['children'][0])
        arguments = GenericList.from_json(jsn['children'][1])
        return FunctionCall(jsn['pos'], function, arguments)

    def __init__(self, pos, function, arguments):
        super().__init__(pos)
        self.__function = function
        self.__arguments = arguments

    # Returns the name of the function the call was made to
    def function(self):
        return self.__function.to_s()

    # Returns the AST nodes containing the arguments to this function
    def arguments(self):
        pass

class Parentheses(Expression):
    CODE = "242000"
    LABEL = "ParenExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Parentheses.CODE)
        expr = Node.from_json(jsn['children'][0])
        return Parentheses(jsn['pos'], expr)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def expr(self):
        return self.__expr

    def to_s(self):
        return "(%s)" % self.__expr.to_s()

# I really have no idea what the point of this node is? Seems to contain nodes
# whose presence is otherwise optional. In the case those nodes are missing, a
# None can be found instead.
# From observation, it only ever seems to contain one item, followed by a ;
class Some(Node):
    CODE = "290100"
    LABEL = "Some"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Some.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        print(children)
        assert len(children) == 2
        assert isinstance(children[1], GenericString)
        assert children[1].to_s() == ";"
        return Some(jsn['pos'], children[0], children[1])

    def __init__(self, pos, expr, semicolon):
        super().__init__(pos)
        self.__expr = expr
        self.__semicolon = semicolon

class Unary(Expression):
    CODE = "241000"
    LABEL = "Unary"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Unary.CODE)
        operand = Node.from_json(jsn['children'][0])
        operator = GenericString.from_json(jsn['children'][1])
        return Unary(jsn['pos'], operand, operator)

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class Binary(Expression):
    CODE = "241100"
    LABEL = "Binary"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Binary.CODE)
        left = Node.from_json(jsn['children'][0])
        op = GenericString.from_json(jsn['children'][1])
        right = Node.from_json(jsn['children'][2])
        return Binary(jsn['pos'], left, op, right)

    def __init__(self, pos, left, op, right):
        super().__init__(pos)
        self.__left = left
        self.__op = op
        self.__right = right

    def op(self):
        return self.__op.label()

class ReturnExpr(Expression):
    CODE = "280200"
    LABEL = "ReturnExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], ReturnExpr.CODE)
        return ReturnExpr(jsn['pos'],\
                          Node.from_json(jsn['children'][0]))

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def to_s(self):
        return "return %s" % self.__expr.to_s()
<commit_msg>fixed Some<commit_after>from basic import *

# Base class used by all AST nodes deemed to represent a C expression
class Expression(Node):
    pass

# Not entirely sure what this is meant to be?
# It has no children, so either it ignores the type whose size it measures, or
# it does something entirely different.
class SizeOfType(Expression):
    CODE = "241600"
    LABEL = "SizeOfType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], SizeOfType.CODE)
        assert not jsn['children']
        return SizeOfType(jsn['pos'])

class Assignment(Expression):
    CODE = "240700"
    LABEL = "Assignment"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Assignment.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 3
        assert isinstance(children[1], GenericString)
        return Assignment(jsn['pos'], children[0], children[2])

    def __init__(self, pos, lhs, rhs):
        super().__init__(pos)
        self.__lhs = lhs
        self.__rhs = rhs

# This seems to just represent Postfix expressions? This has nothing to do with
# Infix expression?
class Infix(Expression):
    CODE = "240900"
    LABEL = "Infix"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Infix.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 2
        return Infix(jsn['pos'], children[0], children[1])

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class Postfix(Expression):
    CODE = "240800"
    LABEL = "Postfix"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Postfix.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 2
        assert isinstance(children[1], GenericString)
        return Postfix(jsn['pos'], children[0], children[1])

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class RecordPtAccess(Expression):
    CODE = "241400"
    LABEL = "RecordPtAccess"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], RecordPtAccess.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert isinstance(children[1], GenericString)
        return RecordPtAccess(jsn['pos'],\
                              children[0],
                              children[1])

    def __init__(self, pos, expr, member):
        super().__init__(pos)
        self.__expr = expr
        self.__member = member

class Identity(Expression):
    CODE = "240100"
    LABEL = "Ident"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Identity.CODE)
        of = GenericString.from_json(jsn['children'][0])
        return Identity(jsn['pos'], of)

    def __init__(self, pos, of):
        super().__init__(pos)
        self.__of = of

    def of(self):
        return self.__of.read()

    def to_s(self):
        return self.__of.read()

class Constant(Expression):
    CODE = "240200"
    LABEL = "Constant"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Constant.CODE)
        return Constant(jsn['pos'], jsn['label'])

    def __init__(self, pos, value):
        super().__init__(pos)
        self.__value = value

    def value(self):
        return self.__value

    def to_s(self):
        return str(self.__value)

class Cast(Expression):
    CODE = "241700"
    LABEL = "Cast"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Cast.CODE)
        expr = Node.from_json(jsn['children'][0])
        return Cast(jsn['pos'], expr)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def to_s(self):
        return "(UNKNOWN_TYPE) %s" % self.__expr.to_s()

# What is the difference between Init and Assignment?
class InitExpr(Expression):
    CODE = "360100"
    LABEL = "InitExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], InitExpr.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        return InitExpr(jsn['pos'], children)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

class Ternary(Expression):
    CODE = "240500"
    LABEL = "CondExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Ternary.CODE)
        return Ternary(jsn['pos'],\
                       Node.from_json(jsn['children'][0]),\
                       Node.from_json(jsn['children'][1]),\
                       Node.from_json(jsn['children'][2]))

    def __init__(self, pos, cond, then, els):
        super().__init__(pos)
        self.__cond = cond
        self.__then = then
        self.__els = els

class FunctionCall(Expression):
    CODE = "240400"
    LABEL = "FunCall"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], FunctionCall.CODE)
        function = Identity.from_json(jsn['children'][0])
        arguments = GenericList.from_json(jsn['children'][1])
        return FunctionCall(jsn['pos'], function, arguments)

    def __init__(self, pos, function, arguments):
        super().__init__(pos)
        self.__function = function
        self.__arguments = arguments

    # Returns the name of the function the call was made to
    def function(self):
        return self.__function.to_s()

    # Returns the AST nodes containing the arguments to this function
    def arguments(self):
        pass

class Parentheses(Expression):
    CODE = "242000"
    LABEL = "ParenExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Parentheses.CODE)
        expr = Node.from_json(jsn['children'][0])
        return Parentheses(jsn['pos'], expr)

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def expr(self):
        return self.__expr

    def to_s(self):
        return "(%s)" % self.__expr.to_s()

# I really have no idea what the point of this node is? Seems to contain nodes
# whose presence is otherwise optional. In the case those nodes are missing, a
# None can be found instead.
# From observation, it only ever seems to contain one item, followed by a ;
class Some(Node):
    CODE = "290100"
    LABEL = "Some"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Some.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) <= 2
        expr = children[0]
        if len(children) == 2:
            semi_colon = children[1]
            assert isinstance(semi_colon, GenericString)
            assert semi_colon.to_s() == ";"
        else:
            semi_colon = None
        return Some(jsn['pos'], expr, semi_colon)

    def __init__(self, pos, expr, semicolon):
        super().__init__(pos)
        self.__expr = expr
        self.__semicolon = semicolon

class Unary(Expression):
    CODE = "241000"
    LABEL = "Unary"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Unary.CODE)
        operand = Node.from_json(jsn['children'][0])
        operator = GenericString.from_json(jsn['children'][1])
        return Unary(jsn['pos'], operand, operator)

    def __init__(self, pos, operand, operator):
        super().__init__(pos)
        self.__operand = operand
        self.__operator = operator

class Binary(Expression):
    CODE = "241100"
    LABEL = "Binary"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Binary.CODE)
        left = Node.from_json(jsn['children'][0])
        op = GenericString.from_json(jsn['children'][1])
        right = Node.from_json(jsn['children'][2])
        return Binary(jsn['pos'], left, op, right)

    def __init__(self, pos, left, op, right):
        super().__init__(pos)
        self.__left = left
        self.__op = op
        self.__right = right

    def op(self):
        return self.__op.label()

class ReturnExpr(Expression):
    CODE = "280200"
    LABEL = "ReturnExpr"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], ReturnExpr.CODE)
        return ReturnExpr(jsn['pos'],\
                          Node.from_json(jsn['children'][0]))

    def __init__(self, pos, expr):
        super().__init__(pos)
        self.__expr = expr

    def to_s(self):
        return "return %s" % self.__expr.to_s()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Tests OPF descriptionTemplate.py-based experiment/sub-experiment pair"""

import os
import pprint
import sys
import unittest2 as unittest

from nupic import NUPIC_ROOT
from nupic.frameworks.opf.opfhelpers import (
  loadExperimentDescriptionScriptFromDir,
  getExperimentDescriptionInterfaceFromModule
)
from nupic.support.unittesthelpers.testcasebase import (
  TestCaseBase as HelperTestCaseBase)

# Our __main__ entry block sets this to an instance of MyTestEnvironment()
g_myEnv = None
g_debug = False

class MyTestEnvironment(object):

  def __init__(self):

    nupicDir = os.path.join(NUPIC_ROOT, "..")

    examplesDir = os.path.join(nupicDir, "examples")

    _debugOut("examplesDir=<%s>" % (examplesDir,))

    assert os.path.exists(examplesDir), \
           "%s is not present in filesystem" % examplesDir

    # This is where we find OPF binaries (e.g., run_opf_experiment.py, etc.)
    # In the autobuild, it is a read-only directory
    self.__opfBinDir = os.path.join(nupicDir, "scripts")
    assert os.path.exists(self.__opfBinDir), \
           "%s is not present in filesystem" % self.__opfBinDir
    _debugOut("self.__opfBinDir=<%s>" % self.__opfBinDir)

    # Where this script is running from (our autotest counterpart may have
    # copied it from its original location)
    self.__testRunDir = os.path.abspath(os.path.dirname(__file__))
    _debugOut("self.__testRunDir=<%s>" % self.__testRunDir)

    # Parent directory of our private OPF experiments
    self.__opfExperimentsParentDir = os.path.join(self.__testRunDir,
                                                  "experiments")
    assert os.path.exists(self.__opfExperimentsParentDir), \
           "%s is not present in filesystem" % self.__opfExperimentsParentDir
    _debugOut("self.__opfExperimentsParentDir=<%s>"
        % self.__opfExperimentsParentDir)

  def getOpfRunExperimentPyPath(self):
    return os.path.join(self.__opfBinDir, "run_opf_experiment.py")

  def getOpfExperimentPath(self, experimentName):
    """
    experimentName:     e.g., "gym"; this string will be used to form
                        a directory path to the experiment.

    Returns:            absolute path to the experiment directory
    """
    path = os.path.join(self.__opfExperimentsParentDir, experimentName)
    assert os.path.isdir(path), \
      "Experiment path %s doesn't exist or is not a directory" % (path,)
    return path

class MyTestCaseBase(HelperTestCaseBase):

  def setUp(self):
    """ Method called to prepare the test fixture. This is called immediately
    before calling the test method; any exception raised by this method will be
    considered an error rather than a test failure. The default implementation
    does nothing.
    """
    global g_myEnv
    if not g_myEnv:
      # Setup environment
      g_myEnv = MyTestEnvironment()

  def tearDown(self):
    """ Method called immediately after the test method has been called and the
    result recorded. This is called even if the test method raised an exception,
    so the implementation in subclasses may need to be particularly careful
    about checking internal state. Any exception raised by this method will be
    considered an error rather than a test failure. This method will only be
    called if the setUp() succeeds, regardless of the outcome of the test
    method. The default implementation does nothing.
    """
    # Reset our log items
    self.resetExtraLogItems()

  def shortDescription(self):
    """ Override to force unittest framework to use test method names instead
    of docstrings in the report.
    """
    return None

  def executePositiveOpfExperiment(self, experimentName, short=False):
    """ Executes a positive OPF RunExperiment test as a subprocess and validates
    its exit status.

    experimentName:     e.g., "gym"; this string will be used to form
                        a directory path to the experiment.

    short:              if True, attempt to run the experiment with --testMode
                        flag turned on, which causes all inference and training
                        iteration counts to be overridden with small counts.

    Returns:            result from _executeExternalCmdAndReapOutputs
    """

    opfRunner = g_myEnv.getOpfRunExperimentPyPath()
    opfExpDir = g_myEnv.getOpfExperimentPath(experimentName)

    r = self.__executePositiveRunExperimentTest(runnerPath=opfRunner,
                                                experimentDirPath=opfExpDir,
                                                short=short)
    return r

  def __executePositiveRunExperimentTest(self,
                                        runnerPath,
                                        experimentDirPath,
                                        customOptions=[],
                                        short=False):
    """ Executes a positive RunExperiment.py test and performs
    basic validation

    runnerPath:         experiment running (LPF or OPF RunExperiment.py path)

    experimentDirPath:  directory containing the description.py file of interest

    short:              if True, attempt to run the experiment with --testMode
                        flag turned on, which causes all inference and training
                        iteration counts to be overridden with small counts.
                        NOTE: if the (possibly aggregated) dataset has fewer
                        rows than the count overrides, then an LPF experiment
                        will fail.

    Returns:            result from _executeExternalCmdAndReapOutputs
    """
    #----------------------------------------
    # Set up args
    command = [
      "python",
      runnerPath,
      experimentDirPath,
    ]

    command.extend(customOptions)

    if short:
      command.append("--testMode")

    self.addExtraLogItem({'command':command})

    #----------------------------------------
    # Execute RunExperiment.py as subprocess and collect results
    r = _executeExternalCmdAndReapOutputs(command)
    self.addExtraLogItem({'result':r})

    _debugOut(("_executeExternalCmdAndReapOutputs(%s)=%s") % (command, r))

    #----------------------------------------
    # Check subprocess exit status
    self.assertEqual(r['exitStatus'], 0,
                     ("Expected status = 0 from %s; got: %s") % \
                        (runnerPath, r['exitStatus'],))

    self.resetExtraLogItems()

    return r

class PositiveTests(MyTestCaseBase):

  #========================
  def test_sub_experiment_override(self):
    expDir = g_myEnv.getOpfExperimentPath("gym")
    module = loadExperimentDescriptionScriptFromDir(expDir)

    expIface = getExperimentDescriptionInterfaceFromModule(module)

    modelDesc = expIface.getModelDescription()

    tpActivationThreshold = modelDesc['modelParams'] \
        ['tpParams']['activationThreshold']

    expectedValue = 12
    self.assertEqual(tpActivationThreshold, expectedValue,
                     "Expected tp activationThreshold=%s, but got %s" % (
                      expectedValue, tpActivationThreshold))

  def test_run_sub_experiment(self):
    self.executePositiveOpfExperiment(experimentName="gym", short=True)

################################################################################
# Support functions
################################################################################

def _executeExternalCmdAndReapOutputs(args):
  """
  args:     Args list as defined for the args parameter in subprocess.Popen()

  Returns:  result dicionary:
              {
                'exitStatus':<exit-status-of-external-command>,
                'stdoutData':"string",
                'stderrData':"string"
              }
  """
  import subprocess

  _debugOut(("Starting...\n<%s>") % \
                (args,))

  p = subprocess.Popen(args,
                       env=os.environ,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE)
  _debugOut(("Process started for <%s>") % (args,))

  (stdoutData, stderrData) = p.communicate()
  _debugOut(("Process completed for <%s>: exit status=%s, " + \
             "stdoutDataType=%s, stdoutData=<%s>, stderrData=<%s>") % \
                (args, p.returncode, type(stdoutData), stdoutData, stderrData))

  result = dict(
    exitStatus = p.returncode,
    stdoutData = stdoutData,
    stderrData = stderrData,
  )

  _debugOut(("args: <%s>: result:\n%s") % \
                (args, pprint.pformat(result, indent=4)))

  return result

def _debugOut(msg):
  if g_debug:
    callerTraceback = whoisCallersCaller()
    print "OPF TestDescriptionTemplate (f=%s;line=%s): %s" % \
            (callerTraceback.function, callerTraceback.lineno, msg,)
    sys.stdout.flush()

def whoisCallersCaller():
  """
  Returns: Traceback namedtuple for our caller's caller
  """
  import inspect

  frameObj = inspect.stack()[2][0]

  return inspect.getframeinfo(frameObj)

if __name__ == "__main__":
  g_myEnv = MyTestEnvironment()

  unittest.longMessage = True
  unittest.main()
<commit_msg>Update opf_description_template_test.py<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Tests OPF descriptionTemplate.py-based experiment/sub-experiment pair"""

import os
import pprint
import sys
import unittest2 as unittest
from pkg_resources import resource_filename

from nupic.frameworks.opf.opfhelpers import (
  loadExperimentDescriptionScriptFromDir,
  getExperimentDescriptionInterfaceFromModule
)
from nupic.support.unittesthelpers.testcasebase import (
  TestCaseBase as HelperTestCaseBase)

# Our __main__ entry block sets this to an instance of MyTestEnvironment()
g_myEnv = None
g_debug = False

class MyTestEnvironment(object):

  def __init__(self):

    nupicDir = resource_filename("nupic", "")

    examplesDir = os.path.join(nupicDir, "examples")

    _debugOut("examplesDir=<%s>" % (examplesDir,))

    assert os.path.exists(examplesDir), \
           "%s is not present in filesystem" % examplesDir

    # This is where we find OPF binaries (e.g., run_opf_experiment.py, etc.)
    # In the autobuild, it is a read-only directory
    self.__opfBinDir = os.path.join(nupicDir, "scripts")
    assert os.path.exists(self.__opfBinDir), \
           "%s is not present in filesystem" % self.__opfBinDir
    _debugOut("self.__opfBinDir=<%s>" % self.__opfBinDir)

    # Where this script is running from (our autotest counterpart may have
    # copied it from its original location)
    self.__testRunDir = os.path.abspath(os.path.dirname(__file__))
    _debugOut("self.__testRunDir=<%s>" % self.__testRunDir)

    # Parent directory of our private OPF experiments
    self.__opfExperimentsParentDir = os.path.join(self.__testRunDir,
                                                  "experiments")
    assert os.path.exists(self.__opfExperimentsParentDir), \
           "%s is not present in filesystem" % self.__opfExperimentsParentDir
    _debugOut("self.__opfExperimentsParentDir=<%s>"
        % self.__opfExperimentsParentDir)

  def getOpfRunExperimentPyPath(self):
    return os.path.join(self.__opfBinDir, "run_opf_experiment.py")

  def getOpfExperimentPath(self, experimentName):
    """
    experimentName:     e.g., "gym"; this string will be used to form
                        a directory path to the experiment.

    Returns:            absolute path to the experiment directory
    """
    path = os.path.join(self.__opfExperimentsParentDir, experimentName)
    assert os.path.isdir(path), \
      "Experiment path %s doesn't exist or is not a directory" % (path,)
    return path

class MyTestCaseBase(HelperTestCaseBase):

  def setUp(self):
    """ Method called to prepare the test fixture. This is called immediately
    before calling the test method; any exception raised by this method will be
    considered an error rather than a test failure. The default implementation
    does nothing.
    """
    global g_myEnv
    if not g_myEnv:
      # Setup environment
      g_myEnv = MyTestEnvironment()

  def tearDown(self):
    """ Method called immediately after the test method has been called and the
    result recorded. This is called even if the test method raised an exception,
    so the implementation in subclasses may need to be particularly careful
    about checking internal state. Any exception raised by this method will be
    considered an error rather than a test failure. This method will only be
    called if the setUp() succeeds, regardless of the outcome of the test
    method. The default implementation does nothing.
    """
    # Reset our log items
    self.resetExtraLogItems()

  def shortDescription(self):
    """ Override to force unittest framework to use test method names instead
    of docstrings in the report.
    """
    return None

  def executePositiveOpfExperiment(self, experimentName, short=False):
    """ Executes a positive OPF RunExperiment test as a subprocess and validates
    its exit status.

    experimentName:     e.g., "gym"; this string will be used to form
                        a directory path to the experiment.

    short:              if True, attempt to run the experiment with --testMode
                        flag turned on, which causes all inference and training
                        iteration counts to be overridden with small counts.

    Returns:            result from _executeExternalCmdAndReapOutputs
    """

    opfRunner = g_myEnv.getOpfRunExperimentPyPath()
    opfExpDir = g_myEnv.getOpfExperimentPath(experimentName)

    r = self.__executePositiveRunExperimentTest(runnerPath=opfRunner,
                                                experimentDirPath=opfExpDir,
                                                short=short)
    return r

  def __executePositiveRunExperimentTest(self,
                                        runnerPath,
                                        experimentDirPath,
                                        customOptions=[],
                                        short=False):
    """ Executes a positive RunExperiment.py test and performs
    basic validation

    runnerPath:         experiment running (LPF or OPF RunExperiment.py path)

    experimentDirPath:  directory containing the description.py file of interest

    short:              if True, attempt to run the experiment with --testMode
                        flag turned on, which causes all inference and training
                        iteration counts to be overridden with small counts.
                        NOTE: if the (possibly aggregated) dataset has fewer
                        rows than the count overrides, then an LPF experiment
                        will fail.

    Returns:            result from _executeExternalCmdAndReapOutputs
    """
    #----------------------------------------
    # Set up args
    command = [
      "python",
      runnerPath,
      experimentDirPath,
    ]

    command.extend(customOptions)

    if short:
      command.append("--testMode")

    self.addExtraLogItem({'command':command})

    #----------------------------------------
    # Execute RunExperiment.py as subprocess and collect results
    r = _executeExternalCmdAndReapOutputs(command)
    self.addExtraLogItem({'result':r})

    _debugOut(("_executeExternalCmdAndReapOutputs(%s)=%s") % (command, r))

    #----------------------------------------
    # Check subprocess exit status
    self.assertEqual(r['exitStatus'], 0,
                     ("Expected status = 0 from %s; got: %s") % \
                        (runnerPath, r['exitStatus'],))

    self.resetExtraLogItems()

    return r

class PositiveTests(MyTestCaseBase):

  #========================
  def test_sub_experiment_override(self):
    expDir = g_myEnv.getOpfExperimentPath("gym")
    module = loadExperimentDescriptionScriptFromDir(expDir)

    expIface = getExperimentDescriptionInterfaceFromModule(module)

    modelDesc = expIface.getModelDescription()

    tpActivationThreshold = modelDesc['modelParams'] \
        ['tpParams']['activationThreshold']

    expectedValue = 12
    self.assertEqual(tpActivationThreshold, expectedValue,
                     "Expected tp activationThreshold=%s, but got %s" % (
                      expectedValue, tpActivationThreshold))

  def test_run_sub_experiment(self):
    self.executePositiveOpfExperiment(experimentName="gym", short=True)

################################################################################
# Support functions
################################################################################

def _executeExternalCmdAndReapOutputs(args):
  """
  args:     Args list as defined for the args parameter in subprocess.Popen()

  Returns:  result dicionary:
              {
                'exitStatus':<exit-status-of-external-command>,
                'stdoutData':"string",
                'stderrData':"string"
              }
  """
  import subprocess

  _debugOut(("Starting...\n<%s>") % \
                (args,))

  p = subprocess.Popen(args,
                       env=os.environ,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE)
  _debugOut(("Process started for <%s>") % (args,))

  (stdoutData, stderrData) = p.communicate()
  _debugOut(("Process completed for <%s>: exit status=%s, " + \
             "stdoutDataType=%s, stdoutData=<%s>, stderrData=<%s>") % \
                (args, p.returncode, type(stdoutData), stdoutData, stderrData))

  result = dict(
    exitStatus = p.returncode,
    stdoutData = stdoutData,
    stderrData = stderrData,
  )

  _debugOut(("args: <%s>: result:\n%s") % \
                (args, pprint.pformat(result, indent=4)))

  return result

def _debugOut(msg):
  if g_debug:
    callerTraceback = whoisCallersCaller()
    print "OPF TestDescriptionTemplate (f=%s;line=%s): %s" % \
            (callerTraceback.function, callerTraceback.lineno, msg,)
    sys.stdout.flush()

def whoisCallersCaller():
  """
  Returns: Traceback namedtuple for our caller's caller
  """
  import inspect

  frameObj = inspect.stack()[2][0]

  return inspect.getframeinfo(frameObj)

if __name__ == "__main__":
  g_myEnv = MyTestEnvironment()

  unittest.longMessage = True
  unittest.main()
<|endoftext|>


<commit_before>import re
import numbers

class Solution(object):
    """ PhreeqPy Solution Class """

    def __init__(self, phreeqpython, number):
        self.pp = phreeqpython
        self.factor = 1
        self.number = number

    def copy(self):
        """ Create a new copy, with unique solution number, from this solution """
        return self.pp.copy_solution(self.number)

    def add(self, element, mmol):
        self.pp.change_solution(self.number, {element:mmol})
    def remove(self, element, mmol):
        mmol = -mmol
        self.pp.change_solution(self.number, {element:mmol})
    def remove_fraction(self, species, fraction):
        current = self.total(species)
        to_remove = 1000 * current * fraction
        self.remove(species, to_remove)

    # this function can precipitate and dissolve!
    def saturate(self, phase, to_si=0, partial_pressure = 10):
        self.pp.equalize_solution(self.number, phase, to_si, partial_pressure)

    # this function can only precipitate
    def desaturate(self, phase, to_si=0):
        self.pp.equalize_solution(self.number, phase, to_si, 0)

    # change the ph
    def change_ph(self, to_pH, with_chemical=None):
        # default to NaOH and HCl
        if not with_chemical:
            if to_pH < self.pH:
                # dose HCl to lower pH
                self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, "HCl")
            else:
                # dose NaOH to raise pH
                self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, "NaOH")
        else:
            self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, with_chemical)

    def change_temperature(self, to_temperature):
        self.pp.change_solution_temperature(self.number, to_temperature)

    def total(self, element):
        """ Returns to total of any given species or element (SLOW!) """
        total = 0
        regexp = "(^|[^A-Z])"+element
        for species, amount in self.species.iteritems():
            if re.search(regexp, species):
                total += amount
        return total

    def total_element(self, element):
        """ Returns to total any given element (FAST!) """
        return self.pp.ip.get_total_element(self.number, element)

    def tacc(self,temperature=90):
        """ Calculate the Calcium Carbonate Precipitation Potential (CCPP)"""
        # create temporary solution
        tmp = self.copy()
        # raise temperature
        tmp.change_temperature(temperature)
        ca_pre = tmp.total_element('Ca')
        # use saturate instead of desaturate to allow dissolution in addition to precipitation
        tmp.saturate('Calcite',0.0)
        # calculate tacc
        tacc = ca_pre - tmp.total_element('Ca')
        # cleanup
        self.pp.remove_solutions([tmp.number])
        return tacc * 1000 #mmol

    def si(self, phase):
        """ Calculate the TACC """
        return self.pp.ip.get_si(self.number, phase)

    def forget(self):
        """ remove this solution from VIPhreeqc memory """
        self.pp.remove_solutions([self.number])

    # Magic functions
    def __add__(self, other):
        """ add two solutions """
        if not isinstance(other,Solution):
            raise TypeError("Invalid operation, only addition of two solutions is allowed")
        mixture= {self:self.factor, other:other.factor}
        #print mixture
        mixture = self.pp.mix_solutions({self:self.factor,other:other.factor})
        # reset factors to 1
        self.factor = 1
        other.factor = 1

        return mixture

    def __div__(self, other):
        """ set devision factor """
        if not isinstance(other,numbers.Real):
            raise TypeError("Invalid operation, only divisiion by a number is allowed")
        self.factor = 1/float(other)
        return self

    def __mul__(self, other):
        """ set multiplication factor """
        if not isinstance(other,numbers.Real):
            raise TypeError("Invalid operation, only divisiion by a number is allowed")
        self.factor = float(other)
        return self

    # Accessor methods
    @property
    def pH(self):
        return self.pp.ip.get_ph(self.number)
    @property
    def sc(self):
        return self.pp.ip.get_sc(self.number)
    @property
    def temperature(self):
        return self.pp.ip.get_temperature(self.number)
    @property
    def mass(self):
        return self.pp.ip.get_mass(self.number)
    @property
    def pe(self):
        return self.pp.ip.get_pe(self.number)
    @property
    def phases(self):
        return self.pp.ip.get_phases_si(self.number)
    @property
    def elements(self):
        return self.pp.ip.get_elements_totals(self.number)
    @property
    def species(self):
        return self.pp.ip.get_species_moles(self.number)
    @property
    def species_moles(self):
        return self.pp.ip.get_species_moles(self.number)
    @property
    def species_molalities(self):
        return self.pp.ip.get_species_moles(self.number)

    # vitens TACC90 calculation
    @property
    def tacc90(self):
        return self.tacc(90)

<commit_msg>minor changes, added docstrings<commit_after>import re
import numbers

class Solution(object):
    """ PhreeqPy Solution Class """

    def __init__(self, phreeqpython, number):
        self.pp = phreeqpython
        self.factor = 1
        self.number = number

    def copy(self):
        """ Create a new copy, with unique solution number, from this solution """
        return self.pp.copy_solution(self.number)

    def add(self, element, mmol):
        """ Add a chemical to the solution """
        self.pp.change_solution(self.number, {element:mmol})
    def remove(self, element, mmol):
        """ Remove a chemical from the solution """
        mmol = -mmol
        self.pp.change_solution(self.number, {element:mmol})
    def remove_fraction(self, species, fraction):
        """ Remove a fraction of a chemical from the solution """
        current = self.total(species)
        to_remove = 1000 * current * fraction
        self.remove(species, to_remove)

    def saturate(self, phase, to_si=0, in_phase=10):
        """ Saturate (or desaturate) a phase to the given SI.
        This function can both desaturate a phase through precipitation, or saturate a phase
        through dissolution. The maximum amount that can be dissolved is given by in_phase
        """
        self.pp.equalize_solution(self.number, phase, to_si, in_phase)

    # this function can only precipitate
    def desaturate(self, phase, to_si=0):
        """ Desaturate a phase to the given SI.
        This function can only desaturate a phase through precipitation
        """
        self.pp.equalize_solution(self.number, phase, to_si, 0)

    # change the ph
    def change_ph(self, to_pH, with_chemical=None):
        """ Change the pH of a solution by dosing either HCl and NaOH, or a user supplied acid or base """
        # default to NaOH and HCl
        if not with_chemical:
            if to_pH < self.pH:
                # dose HCl to lower pH
                self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, "HCl")
            else:
                # dose NaOH to raise pH
                self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, "NaOH")
        else:
            self.pp.equalize_solution(self.number, "Fix_pH", -to_pH, 10, with_chemical)

    def change_temperature(self, to_temperature):
        """ Change the temperature of a solution """
        self.pp.change_solution_temperature(self.number, to_temperature)

    def total(self, element):
        """ Returns to total of any given species or element (SLOW!) """
        total = 0
        regexp = "(^|[^A-Z])"+element
        for species, amount in self.species.iteritems():
            if re.search(regexp, species):
                total += amount
        return total

    def total_element(self, element):
        """ Returns to total any given element (FAST!) """
        return self.pp.ip.get_total_element(self.number, element)

    def tacc(self,temperature=90):
        """ Calculate the Calcium Carbonate Precipitation Potential (CCPP)"""
        # create temporary solution
        tmp = self.copy()
        # raise temperature
        tmp.change_temperature(temperature)
        ca_pre = tmp.total_element('Ca')
        # use saturate instead of desaturate to allow dissolution in addition to precipitation
        tmp.saturate('Calcite',0.0)
        # calculate tacc
        tacc = ca_pre - tmp.total_element('Ca')
        # cleanup
        self.pp.remove_solutions([tmp.number])
        return tacc * 1000 #mmol

    def si(self, phase):
        """ Calculate the TACC """
        return self.pp.ip.get_si(self.number, phase)

    def forget(self):
        """ remove this solution from VIPhreeqc memory """
        self.pp.remove_solutions([self.number])

    # Magic functions
    def __add__(self, other):
        """ add two solutions """
        if not isinstance(other,Solution):
            raise TypeError("Invalid operation, only addition of two solutions is allowed")
        mixture= {self:self.factor, other:other.factor}
        #print mixture
        mixture = self.pp.mix_solutions({self:self.factor,other:other.factor})
        # reset factors to 1
        self.factor = 1
        other.factor = 1

        return mixture

    def __div__(self, other):
        """ set devision factor """
        if not isinstance(other,numbers.Real):
            raise TypeError("Invalid operation, only divisiion by a number is allowed")
        self.factor = 1/float(other)
        return self

    def __mul__(self, other):
        """ set multiplication factor """
        if not isinstance(other,numbers.Real):
            raise TypeError("Invalid operation, only divisiion by a number is allowed")
        self.factor = float(other)
        return self

    # Accessor methods
    @property
    def pH(self):
        """ Solution pH """
        return self.pp.ip.get_ph(self.number)
    @property
    def sc(self):
        return self.pp.ip.get_sc(self.number)
    @property
    def temperature(self):
        return self.pp.ip.get_temperature(self.number)
    @property
    def mass(self):
        return self.pp.ip.get_mass(self.number)
    @property
    def pe(self):
        return self.pp.ip.get_pe(self.number)
    @property
    def phases(self):
        return self.pp.ip.get_phases_si(self.number)
    @property
    def elements(self):
        return self.pp.ip.get_elements_totals(self.number)
    @property
    def species(self):
        return self.pp.ip.get_species_moles(self.number)
    @property
    def species_moles(self):
        return self.pp.ip.get_species_moles(self.number)
    @property
    def species_molalities(self):
        return self.pp.ip.get_species_moles(self.number)

    # vitens TACC90 calculation
    @property
    def tacc90(self):
        return self.tacc(90)

<|endoftext|>


<commit_before>"""
Regression tests for a few FK bugs: #1578, #6886
"""

from django.db import models

# If ticket #1578 ever slips back in, these models will not be able to be
# created (the field names being lower-cased versions of their opposite
# classes is important here).

class First(models.Model):
    second = models.IntegerField()

class Second(models.Model):
    first = models.ForeignKey(First, related_name = 'the_first')

# Protect against repetition of #1839, #2415 and #2536.
class Third(models.Model):
    name = models.CharField(max_length=20)
    third = models.ForeignKey('self', null=True, related_name='child_set')

class Parent(models.Model):
    name = models.CharField(max_length=20)
    bestchild = models.ForeignKey('Child', null=True, related_name='favored_by')

class Child(models.Model):
    name = models.CharField(max_length=20)
    parent = models.ForeignKey(Parent)

# Multiple paths to the same model (#7110, #7125)
class Category(models.Model):
    name = models.CharField(max_length=20)

    def __unicode__(self):
        return self.name

class Record(models.Model):
    category = models.ForeignKey(Category)

class Relation(models.Model):
    left = models.ForeignKey(Record, related_name='left_set')
    right = models.ForeignKey(Record, related_name='right_set')

    def __unicode__(self):
        return u"%s - %s" % (self.left.category.name, self.right.category.name)

__test__ = {'API_TESTS':"""
>>> Third.objects.create(id='3', name='An example')
<Third: Third object>
>>> parent = Parent(name = 'fred')
>>> parent.save()
>>> Child.objects.create(name='bam-bam', parent=parent)
<Child: Child object>

#
# Tests of ForeignKey assignment and the related-object cache (see #6886).
#
>>> p = Parent.objects.create(name="Parent")
>>> c = Child.objects.create(name="Child", parent=p)

# Look up the object again so that we get a "fresh" object.
>>> c = Child.objects.get(name="Child")
>>> p = c.parent

# Accessing the related object again returns the exactly same object.
>>> c.parent is p
True

# But if we kill the cache, we get a new object.
>>> del c._parent_cache
>>> c.parent is p
False

# Assigning a new object results in that object getting cached immediately.
>>> p2 = Parent.objects.create(name="Parent 2")
>>> c.parent = p2
>>> c.parent is p2
True

# Assigning None succeeds if field is null=True.
>>> p.bestchild = None
>>> p.bestchild is None
True

# Assigning None fails: Child.parent is null=False.
>>> c.parent = None
Traceback (most recent call last):
    ...
ValueError: Cannot assign None: "Child.parent" does not allow null values.

# You also can't assign an object of the wrong type here
>>> c.parent = First(id=1, second=1)
Traceback (most recent call last):
    ...
ValueError: Cannot assign "<First: First object>": "Child.parent" must be a "Parent" instance.

# Creation using keyword argument should cache the related object.
>>> p = Parent.objects.get(name="Parent")
>>> c = Child(parent=p)
>>> c.parent is p
True

# Creation using keyword argument and unsaved related instance (#8070).
>>> p = Parent()
>>> c = Child(parent=p)
>>> c.parent is p
True

# Creation using attname keyword argument and an id will cause the related
# object to be fetched.
>>> p = Parent.objects.get(name="Parent")
>>> c = Child(parent_id=p.id)
>>> c.parent is p
False
>>> c.parent == p
True

#
# Test of multiple ForeignKeys to the same model (bug #7125).
#
>>> c1 = Category.objects.create(name='First')
>>> c2 = Category.objects.create(name='Second')
>>> c3 = Category.objects.create(name='Third')
>>> r1 = Record.objects.create(category=c1)
>>> r2 = Record.objects.create(category=c1)
>>> r3 = Record.objects.create(category=c2)
>>> r4 = Record.objects.create(category=c2)
>>> r5 = Record.objects.create(category=c3)
>>> r = Relation.objects.create(left=r1, right=r2)
>>> r = Relation.objects.create(left=r3, right=r4)
>>> r = Relation.objects.create(left=r1, right=r3)
>>> r = Relation.objects.create(left=r5, right=r2)
>>> r = Relation.objects.create(left=r3, right=r2)

>>> Relation.objects.filter(left__category__name__in=['First'], right__category__name__in=['Second'])
[<Relation: First - Second>]

>>> Category.objects.filter(record__left_set__right__category__name='Second').order_by('name')
[<Category: First>, <Category: Second>]

"""}
<commit_msg>Added tests for accessing nullable ForeignKey after saving and fetching from the database (refs #8093).<commit_after>"""
Regression tests for a few FK bugs: #1578, #6886
"""

from django.db import models

# If ticket #1578 ever slips back in, these models will not be able to be
# created (the field names being lower-cased versions of their opposite
# classes is important here).

class First(models.Model):
    second = models.IntegerField()

class Second(models.Model):
    first = models.ForeignKey(First, related_name = 'the_first')

# Protect against repetition of #1839, #2415 and #2536.
class Third(models.Model):
    name = models.CharField(max_length=20)
    third = models.ForeignKey('self', null=True, related_name='child_set')

class Parent(models.Model):
    name = models.CharField(max_length=20)
    bestchild = models.ForeignKey('Child', null=True, related_name='favored_by')

class Child(models.Model):
    name = models.CharField(max_length=20)
    parent = models.ForeignKey(Parent)

# Multiple paths to the same model (#7110, #7125)
class Category(models.Model):
    name = models.CharField(max_length=20)

    def __unicode__(self):
        return self.name

class Record(models.Model):
    category = models.ForeignKey(Category)

class Relation(models.Model):
    left = models.ForeignKey(Record, related_name='left_set')
    right = models.ForeignKey(Record, related_name='right_set')

    def __unicode__(self):
        return u"%s - %s" % (self.left.category.name, self.right.category.name)

__test__ = {'API_TESTS':"""
>>> Third.objects.create(id='3', name='An example')
<Third: Third object>
>>> parent = Parent(name = 'fred')
>>> parent.save()
>>> Child.objects.create(name='bam-bam', parent=parent)
<Child: Child object>

#
# Tests of ForeignKey assignment and the related-object cache (see #6886).
#
>>> p = Parent.objects.create(name="Parent")
>>> c = Child.objects.create(name="Child", parent=p)

# Look up the object again so that we get a "fresh" object.
>>> c = Child.objects.get(name="Child")
>>> p = c.parent

# Accessing the related object again returns the exactly same object.
>>> c.parent is p
True

# But if we kill the cache, we get a new object.
>>> del c._parent_cache
>>> c.parent is p
False

# Assigning a new object results in that object getting cached immediately.
>>> p2 = Parent.objects.create(name="Parent 2")
>>> c.parent = p2
>>> c.parent is p2
True

# Assigning None succeeds if field is null=True.
>>> p.bestchild = None
>>> p.bestchild is None
True

# bestchild should still be None after saving.
>>> p.save()
>>> p.bestchild is None
True

# bestchild should still be None after fetching the object again.
>>> p = Parent.objects.get(name="Parent")
>>> p.bestchild is None
True

# Assigning None fails: Child.parent is null=False.
>>> c.parent = None
Traceback (most recent call last):
    ...
ValueError: Cannot assign None: "Child.parent" does not allow null values.

# You also can't assign an object of the wrong type here
>>> c.parent = First(id=1, second=1)
Traceback (most recent call last):
    ...
ValueError: Cannot assign "<First: First object>": "Child.parent" must be a "Parent" instance.

# Creation using keyword argument should cache the related object.
>>> p = Parent.objects.get(name="Parent")
>>> c = Child(parent=p)
>>> c.parent is p
True

# Creation using keyword argument and unsaved related instance (#8070).
>>> p = Parent()
>>> c = Child(parent=p)
>>> c.parent is p
True

# Creation using attname keyword argument and an id will cause the related
# object to be fetched.
>>> p = Parent.objects.get(name="Parent")
>>> c = Child(parent_id=p.id)
>>> c.parent is p
False
>>> c.parent == p
True

#
# Test of multiple ForeignKeys to the same model (bug #7125).
#
>>> c1 = Category.objects.create(name='First')
>>> c2 = Category.objects.create(name='Second')
>>> c3 = Category.objects.create(name='Third')
>>> r1 = Record.objects.create(category=c1)
>>> r2 = Record.objects.create(category=c1)
>>> r3 = Record.objects.create(category=c2)
>>> r4 = Record.objects.create(category=c2)
>>> r5 = Record.objects.create(category=c3)
>>> r = Relation.objects.create(left=r1, right=r2)
>>> r = Relation.objects.create(left=r3, right=r4)
>>> r = Relation.objects.create(left=r1, right=r3)
>>> r = Relation.objects.create(left=r5, right=r2)
>>> r = Relation.objects.create(left=r3, right=r2)

>>> Relation.objects.filter(left__category__name__in=['First'], right__category__name__in=['Second'])
[<Relation: First - Second>]

>>> Category.objects.filter(record__left_set__right__category__name='Second').order_by('name')
[<Category: First>, <Category: Second>]

"""}
<|endoftext|>


<commit_before>#    Copyright 2016-2016 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Base matplotlib plotter module"""
from abc import abstractmethod, ABCMeta
from collections import defaultdict as ddict
import matplotlib.pyplot as plt
from trappy.plotter import AttrConf
from trappy.plotter.Constraint import ConstraintManager
from trappy.plotter.PlotLayout import PlotLayout
from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter
from trappy.plotter.ColorMap import ColorMap

class StaticPlot(AbstractDataPlotter):
    """
    This class uses :mod:`trappy.plotter.Constraint.Constraint` to
    represent different permutations of input parameters. These
    constraints are generated by creating an instance of
    :mod:`trappy.plotter.Constraint.ConstraintManager`.

    :param trace: The input data
    :type trace: :mod:`trappy.trace.FTrace` or :mod:`pandas.DataFrame`, list or single

    :param column: specifies the name of the column to
           be plotted.
    :type column: (str, list(str))

    :param templates: TRAPpy events

        .. note::

                This is not required if a :mod:`pandas.DataFrame` is
                used

    :type templates: :mod:`trappy.base.Base`

    :param filters: Filter the column to be plotted as per the
        specified criteria. For Example:
        ::

            filters =
                    {
                        "pid": [ 3338 ],
                        "cpu": [0, 2, 4],
                    }
    :type filters: dict

    :param per_line: Used to control the number of graphs
        in each graph subplot row
    :type per_line: int

    :param concat: Draw all the pivots on a single graph
    :type concat: bool

    :param permute: Draw one plot for each of the traces specified
    :type permute: bool

    :param drawstyle: This argument is forwarded to the matplotlib
        corresponding :func:`matplotlib.pyplot.plot` call

        drawing style.

        .. note::

            step plots are not currently supported for filled
            graphs

    :param xlim: A tuple representing the upper and lower xlimits
    :type xlim: tuple

    :param ylim: A tuple representing the upper and lower ylimits
    :type ylim: tuple

    :param title: A title describing all the generated plots
    :type title: str

    :param style: Created pre-styled graphs loaded from
        :mod:`trappy.plotter.AttrConf.MPL_STYLE`
    :type style: bool

    :param signals: A string of the type event_name:column
        to indicate the value that needs to be plotted

        .. note::

            - Only one of `signals` or both `templates` and
              `columns` should be specified
            - Signals format won't work for :mod:`pandas.DataFrame`
              input

    :type signals: str

    """
    __metaclass__ = ABCMeta

    def __init__(self, traces, templates, **kwargs):
        self._fig = None
        self._layout = None
        super(StaticPlot, self).__init__(traces=traces,
                                         templates=templates)

        self.set_defaults()

        for key in kwargs:
            if key in AttrConf.ARGS_TO_FORWARD:
                self._attr["args_to_forward"][key] = kwargs[key]
            else:
                self._attr[key] = kwargs[key]

        if "signals" in self._attr:
            self._describe_signals()

        self._check_data()

        if "column" not in self._attr:
            raise RuntimeError("Value Column not specified")

        zip_constraints = not self._attr["permute"]
        self.c_mgr = ConstraintManager(traces, self._attr["column"],
                                       self.templates, self._attr["pivot"],
                                       self._attr["filters"], zip_constraints)

    def savefig(self, *args, **kwargs):
        """Save the plot as a PNG fill. This calls into
        :mod:`matplotlib.figure.savefig`
        """

        if self._fig is None:
            self.view()
        self._fig.savefig(*args, **kwargs)

    @abstractmethod
    def set_defaults(self):
        """Sets the default attrs"""
        self._attr["width"] = AttrConf.WIDTH
        self._attr["length"] = AttrConf.LENGTH
        self._attr["per_line"] = AttrConf.PER_LINE
        self._attr["concat"] = AttrConf.CONCAT
        self._attr["filters"] = {}
        self._attr["style"] = True
        self._attr["permute"] = False
        self._attr["pivot"] = AttrConf.PIVOT
        self._attr["xlim"] = AttrConf.XLIM
        self._attr["ylim"] = AttrConf.XLIM
        self._attr["title"] = AttrConf.TITLE
        self._attr["args_to_forward"] = {}
        self._attr["map_label"] = {}
        self._attr["_legend_handles"] = []
        self._attr["_legend_labels"] = []

    def view(self, test=False):
        """Displays the graph"""

        if test:
            self._attr["style"] = True
            AttrConf.MPL_STYLE["interactive"] = False

        permute = self._attr["permute"] and not self._attr["concat"]
        if self._attr["style"]:
            with plt.rc_context(AttrConf.MPL_STYLE):
                self._resolve(permute, self._attr["concat"])
        else:
            self._resolve(permute, self._attr["concat"])

    def make_title(self, constraint, pivot, permute, concat):
        """Generates a title string for an axis"""
        if concat:
            return str(constraint)
        title = ""
        if permute:
            title += constraint.get_data_name() + ":" + constraint.column
        else:
            if pivot == AttrConf.PIVOT_VAL:
                if isinstance(self._attr["column"], list):
                    title += ", ".join(self._attr["column"])
                else:
                    title += self._attr["column"]
            else:
                title += "{0}: {1}".format(self._attr["pivot"],
                                           self._attr["map_label"].get(pivot, pivot))
        return title

    def add_to_legend(self, series_index, handle, constraint, pivot, concat):
        """
        Add series handles and names to the legend
        A handle is returned from a plot on an axis
        e.g. Line2D from axis.plot()
        """
        self._attr["_legend_handles"][series_index] = handle
        legend_labels = self._attr["_legend_labels"]

        if concat and pivot == AttrConf.PIVOT_VAL:
            legend_labels[series_index] = self._attr["column"]
        elif concat:
            legend_labels[series_index] = "{0}: {1}".format(
                self._attr["pivot"],
                self._attr["map_label"].get(pivot, pivot)
            )
        else:
            legend_labels[series_index] = str(constraint)
            # Remove trace name if there is only one trace to plot
            if not isinstance(self.traces, list):
                legend_labels[series_index] = legend_labels[series_index].replace(
                                                constraint.get_data_name()+":", ""
                                              )

    def _resolve(self, permute, concat):
        """Determine what data to plot on which axis"""
        pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute)
        pivot_vals = list(pivot_vals)

        num_of_axes = len(self.c_mgr) if concat else len_pivots

        # Create a 2D Layout
        self._layout = PlotLayout(
            self._attr["per_line"],
            num_of_axes,
            width=self._attr["width"],
            length=self._attr["length"],
            title=self._attr['title'])

        self._fig = self._layout.get_fig()

        #Determine what constraint to plot and the corresponding pivot value
        if permute:
            legend_len = self.c_mgr._max_len
            pivots = [y for _, y in pivot_vals]
            cp_pairs = [(c, p) for c in self.c_mgr for p in sorted(set(pivots))]
        else:
            legend_len = len_pivots if concat else len(self.c_mgr)
            pivots = pivot_vals
            cp_pairs = [(c, p) for c in self.c_mgr for p in pivots if p in c.result]

        #Initialise legend data and colormap
        self._attr["_legend_handles"] = [None] * legend_len
        self._attr["_legend_labels"] = [None] * legend_len
        self._cmap = ColorMap(legend_len)

        #Group constraints/series with the axis they are to be plotted on
        figure_data = ddict(list)
        for i, (constraint, pivot) in enumerate(cp_pairs):
            axis = self._layout.get_axis(constraint.trace_index if concat else i)
            figure_data[axis].append((constraint, pivot))

        #Plot each axis
        for axis, series_list in figure_data.iteritems():
            self.plot_axis(
                axis,
                series_list,
                permute,
                self._attr["concat"],
                self._attr["args_to_forward"]
            )

        #Add the legend to the figure if more than one signal is plotted
        if legend_len > 1:
            self._fig.legend(self._attr["_legend_handles"],
                             self._attr["_legend_labels"])

        self._layout.finish(num_of_axes)

    def plot_axis(self, axis, series_list, permute, concat, args_to_forward):
        """Internal Method called to plot data (series_list) on a given axis"""
        raise NotImplementedError("Method Not Implemented")
<commit_msg>plotter: StaticPlot: move legend below graph<commit_after>#    Copyright 2016-2016 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Base matplotlib plotter module"""
from abc import abstractmethod, ABCMeta
from collections import defaultdict as ddict
import matplotlib.pyplot as plt
from trappy.plotter import AttrConf
from trappy.plotter.Constraint import ConstraintManager
from trappy.plotter.PlotLayout import PlotLayout
from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter
from trappy.plotter.ColorMap import ColorMap

class StaticPlot(AbstractDataPlotter):
    """
    This class uses :mod:`trappy.plotter.Constraint.Constraint` to
    represent different permutations of input parameters. These
    constraints are generated by creating an instance of
    :mod:`trappy.plotter.Constraint.ConstraintManager`.

    :param trace: The input data
    :type trace: :mod:`trappy.trace.FTrace` or :mod:`pandas.DataFrame`, list or single

    :param column: specifies the name of the column to
           be plotted.
    :type column: (str, list(str))

    :param templates: TRAPpy events

        .. note::

                This is not required if a :mod:`pandas.DataFrame` is
                used

    :type templates: :mod:`trappy.base.Base`

    :param filters: Filter the column to be plotted as per the
        specified criteria. For Example:
        ::

            filters =
                    {
                        "pid": [ 3338 ],
                        "cpu": [0, 2, 4],
                    }
    :type filters: dict

    :param per_line: Used to control the number of graphs
        in each graph subplot row
    :type per_line: int

    :param concat: Draw all the pivots on a single graph
    :type concat: bool

    :param permute: Draw one plot for each of the traces specified
    :type permute: bool

    :param drawstyle: This argument is forwarded to the matplotlib
        corresponding :func:`matplotlib.pyplot.plot` call

        drawing style.

        .. note::

            step plots are not currently supported for filled
            graphs

    :param xlim: A tuple representing the upper and lower xlimits
    :type xlim: tuple

    :param ylim: A tuple representing the upper and lower ylimits
    :type ylim: tuple

    :param title: A title describing all the generated plots
    :type title: str

    :param style: Created pre-styled graphs loaded from
        :mod:`trappy.plotter.AttrConf.MPL_STYLE`
    :type style: bool

    :param signals: A string of the type event_name:column
        to indicate the value that needs to be plotted

        .. note::

            - Only one of `signals` or both `templates` and
              `columns` should be specified
            - Signals format won't work for :mod:`pandas.DataFrame`
              input

    :type signals: str

    """
    __metaclass__ = ABCMeta

    def __init__(self, traces, templates, **kwargs):
        self._fig = None
        self._layout = None
        super(StaticPlot, self).__init__(traces=traces,
                                         templates=templates)

        self.set_defaults()

        for key in kwargs:
            if key in AttrConf.ARGS_TO_FORWARD:
                self._attr["args_to_forward"][key] = kwargs[key]
            else:
                self._attr[key] = kwargs[key]

        if "signals" in self._attr:
            self._describe_signals()

        self._check_data()

        if "column" not in self._attr:
            raise RuntimeError("Value Column not specified")

        zip_constraints = not self._attr["permute"]
        self.c_mgr = ConstraintManager(traces, self._attr["column"],
                                       self.templates, self._attr["pivot"],
                                       self._attr["filters"], zip_constraints)

    def savefig(self, *args, **kwargs):
        """Save the plot as a PNG fill. This calls into
        :mod:`matplotlib.figure.savefig`
        """

        if self._fig is None:
            self.view()
        self._fig.savefig(*args, **kwargs)

    @abstractmethod
    def set_defaults(self):
        """Sets the default attrs"""
        self._attr["width"] = AttrConf.WIDTH
        self._attr["length"] = AttrConf.LENGTH
        self._attr["per_line"] = AttrConf.PER_LINE
        self._attr["concat"] = AttrConf.CONCAT
        self._attr["filters"] = {}
        self._attr["style"] = True
        self._attr["permute"] = False
        self._attr["pivot"] = AttrConf.PIVOT
        self._attr["xlim"] = AttrConf.XLIM
        self._attr["ylim"] = AttrConf.XLIM
        self._attr["title"] = AttrConf.TITLE
        self._attr["args_to_forward"] = {}
        self._attr["map_label"] = {}
        self._attr["_legend_handles"] = []
        self._attr["_legend_labels"] = []

    def view(self, test=False):
        """Displays the graph"""

        if test:
            self._attr["style"] = True
            AttrConf.MPL_STYLE["interactive"] = False

        permute = self._attr["permute"] and not self._attr["concat"]
        if self._attr["style"]:
            with plt.rc_context(AttrConf.MPL_STYLE):
                self._resolve(permute, self._attr["concat"])
        else:
            self._resolve(permute, self._attr["concat"])

    def make_title(self, constraint, pivot, permute, concat):
        """Generates a title string for an axis"""
        if concat:
            return str(constraint)
        title = ""
        if permute:
            title += constraint.get_data_name() + ":" + constraint.column
        else:
            if pivot == AttrConf.PIVOT_VAL:
                if isinstance(self._attr["column"], list):
                    title += ", ".join(self._attr["column"])
                else:
                    title += self._attr["column"]
            else:
                title += "{0}: {1}".format(self._attr["pivot"],
                                           self._attr["map_label"].get(pivot, pivot))
        return title

    def add_to_legend(self, series_index, handle, constraint, pivot, concat):
        """
        Add series handles and names to the legend
        A handle is returned from a plot on an axis
        e.g. Line2D from axis.plot()
        """
        self._attr["_legend_handles"][series_index] = handle
        legend_labels = self._attr["_legend_labels"]

        if concat and pivot == AttrConf.PIVOT_VAL:
            legend_labels[series_index] = self._attr["column"]
        elif concat:
            legend_labels[series_index] = "{0}: {1}".format(
                self._attr["pivot"],
                self._attr["map_label"].get(pivot, pivot)
            )
        else:
            legend_labels[series_index] = str(constraint)
            # Remove trace name if there is only one trace to plot
            if not isinstance(self.traces, list):
                legend_labels[series_index] = legend_labels[series_index].replace(
                                                constraint.get_data_name()+":", ""
                                              )

    def _resolve(self, permute, concat):
        """Determine what data to plot on which axis"""
        pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute)
        pivot_vals = list(pivot_vals)

        num_of_axes = len(self.c_mgr) if concat else len_pivots

        # Create a 2D Layout
        self._layout = PlotLayout(
            self._attr["per_line"],
            num_of_axes,
            width=self._attr["width"],
            length=self._attr["length"],
            title=self._attr['title'])

        self._fig = self._layout.get_fig()

        #Determine what constraint to plot and the corresponding pivot value
        if permute:
            legend_len = self.c_mgr._max_len
            pivots = [y for _, y in pivot_vals]
            cp_pairs = [(c, p) for c in self.c_mgr for p in sorted(set(pivots))]
        else:
            legend_len = len_pivots if concat else len(self.c_mgr)
            pivots = pivot_vals
            cp_pairs = [(c, p) for c in self.c_mgr for p in pivots if p in c.result]

        #Initialise legend data and colormap
        self._attr["_legend_handles"] = [None] * legend_len
        self._attr["_legend_labels"] = [None] * legend_len
        self._cmap = ColorMap(legend_len)

        #Group constraints/series with the axis they are to be plotted on
        figure_data = ddict(list)
        for i, (constraint, pivot) in enumerate(cp_pairs):
            axis = self._layout.get_axis(constraint.trace_index if concat else i)
            figure_data[axis].append((constraint, pivot))

        #Plot each axis
        for axis, series_list in figure_data.iteritems():
            self.plot_axis(
                axis,
                series_list,
                permute,
                self._attr["concat"],
                self._attr["args_to_forward"]
            )

        #Add the legend to the figure if more than one signal is plotted
        if legend_len > 1:
            self._fig.legend(self._attr["_legend_handles"],
                             self._attr["_legend_labels"],
                             loc='lower center',
                             ncol=3,
                             borderaxespad=0.)

        self._layout.finish(num_of_axes)

    def plot_axis(self, axis, series_list, permute, concat, args_to_forward):
        """Internal Method called to plot data (series_list) on a given axis"""
        raise NotImplementedError("Method Not Implemented")
<|endoftext|>