<commit_before>#! /usr/bin/env python

import os
import logging
import configparser
import random
import warnings

import mann.network
import mann.network_agent
import mann.helper
import mann.lens_in_writer

# from mann import network
# from mann import network_agent
# from mann import helper
# from mann import lens_in_writer

here = os.path.abspath(os.path.dirname(__file__))

# set up logging to file - see previous section for more details
logging_format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
logging.basicConfig(level=logging.DEBUG,
                    format=logging_format,
                    datefmt='%m-%d %H:%M',
                    filename=os.path.join(here, 'output', 'myapp.log'),
                    filemode='w')

# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)

formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')

# tell the handler to use this format
# set a format which is simpler for console use
console.setFormatter(formatter)

# add the handler to the root logger
logging.getLogger('').addHandler(console)

# Now, we can log to the root logger, or any other logger. First the root..
logging.info('Logger created in main()')

# Now, define a couple of other loggers which might represent areas in your
# application:

logger1 = logging.getLogger(os.path.join(here, 'myapp.area1'))
logger2 = logging.getLogger(os.path.join(here, 'myapp.area2'))

# setting up the configparser
config = configparser.ConfigParser()
config.read(os.path.join(here, 'config.ini'))

def random_select_and_update(network_of_agents):
    # randomly select nodes from network_of_agents
    # select num_update number of the nodes for update
    num_update = config.getint('ModelParameters',
                               'NumberOfAgentsToUpdatePerTimeTick')
    agents_for_update = network_of_agents.sample_network(num_update)
    print('agents for update: ', agents_for_update)
    print('key of agent for update')
    print(network_of_agents.G.nodes()[agents_for_update[0].get_key()])

    # update agents who were selected
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])

        lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                                   'UpdateFromInflInFile')

        agent_ex_file_dir = here + '/' + config.get('LENSParameters',
                                                    'AgentExFile')

        infl_ex_file_dir = here + '/' + config.get('LENSParameters',
                                                   'InflExFile')

        agent_state_out_file_dir = here + '/' + config.get('LENSParameters',
                                                           'NewAgentStateFile')

        selected_agent.update_agent_state(
            'default',
            lens_in_file=lens_in_file_dir,
            agent_ex_file=agent_ex_file_dir,
            infl_ex_file=infl_ex_file_dir,
            agent_state_out_file=agent_state_out_file_dir)

def update_simultaneous(network_of_agents, num_agents_update):
    """Simultaneously updates agents

    :param network_of_agents: NetworkX graph of agents
    :type network_of_agents: NetworkX graph

    :parm num_agents_update: Number of agents that will be picked for update
    :type num_agents_update: int

    iterates through each node in the network and calculates a new state value
    that will be saved to a temp variable.  It will then iterate though the
    network again to update each node to the temp variable
    """
    agents_for_update = network_of_agents.sample_network(num_agents_update)
    print('agents for update: ', agents_for_update)
    print('key of agent for update')
    for agent_update_key in agents_for_update:
        print(network_of_agents.G.nodes()[agent_update_key.get_key()])

    lens_in_file_dir = os.path.join(here, config.get('LENSParameters',
                                                     'UpdateFromInflInFile'))
    lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                               'UpdateFromInflInFile')

    infl_ex_file_dir = os.path.join(here, config.get('LENSParameters',
                                                     'InflExFile'))

    agent_state_out_file_dir = os.path.join(here,
                                            config.get('LENSParameters',
                                                       'NewAgentStateFile'))

    print('printing network_of_agents.G nodes')
    for node in network_of_agents.G:
        print(node)

    # save to temp state before looping to sim update
    lens_in_writer_helper = mann.lens_in_writer.LensInWriterHelper()
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])
        print(selected_agent.temp_new_state)
        assert selected_agent.temp_new_state is None

        try:
            random_predecessor_id = random.sample(selected_agent.predecessors,
                                                  1)
        except ValueError:
            # if agent has no predecessor we skip it
            continue
        else:
            predecessor_picked = random_predecessor_id[0]

            print(type(predecessor_picked))

            print(predecessor_picked)

            write_str = lens_in_writer_helper.generate_lens_recurrent_attitude(
                mann.helper.convert_list_to_delim_str(selected_agent.state,
                                                      delim=' '),
                mann.helper.convert_list_to_delim_str(predecessor_picked.state,
                                                      delim=' '))
            lens_in_writer_helper.write_in_file(infl_ex_file_dir, write_str)

            print(lens_in_file_dir)
            selected_agent.call_lens(lens_in_file_dir)

            new_state_values = selected_agent.\
                get_new_state_values_from_out_file(
                    agent_state_out_file_dir,
                    'agent_type param is not used ... yet')
            selected_agent.temp_new_state = new_state_values

    # simultaneous update
    for selected_agent in agents_for_update:
        if len(selected_agent.predecessors) > 0:
            assert (selected_agent.temp_new_state is not None)
            selected_agent.state = selected_agent.temp_new_state
            selected_agent.temp_new_state = None
        else:
            assert len(selected_agent.predecessors) == 0
            warnings_str = "Pedecessors for Agent {}: {}".format(
                selected_agent.agent_id, selected_agent.predecessors)
            warnings.warn(warnings_str)

def step(time_tick, network_of_agents, agent_type):
    logger1.debug('STEP TIME TICK: %s', str(time_tick))

    logger1.debug('Begin random select and update network of agents')

    update_type = config.get('ModelParameters', 'UpdateType')

    if update_type == 'sequential':
        random_select_and_update(network_of_agents)
    elif update_type == 'simultaneous':
        print("Performing a simultaneous update.")
        update_simultaneous(network_of_agents,
                            config.getint(
                                'ModelParameters',
                                'NumberOfAgentsToUpdatePerTimeTick'))
    else:
        raise ValueError('Unknown simulation update type')

    network_agent_step_time_dir = os.path.join(here, 'output',
                                               'network_of_agents.pout')

    network_of_agents.write_network_agent_step_info(
        time_tick, network_agent_step_time_dir, 'a')
    logger1.debug('Time ticks %s values appended to %s',
                  str(time_tick),
                  network_agent_step_time_dir)

def main():
    logger1.info('In main.main()')
    logger1.info('Starting Mulit Agent Neural Network (MANN)')

    # creating n number of agents
    n = config.getint('NetworkParameters', 'NumberOfAgents')
    logger1.debug('Number of agents to create: %s', str(n))

    # probablity for edge creation [0, 1]
    p = config.getfloat('NetworkParameters', 'ProbEdgeCreation')
    logger1.debug('Probablity for edge creation: %s', str(p))

    # Create Erdos-Renyi graph
    my_network = mann.network.DirectedFastGNPRandomGraph(n, p)

    # print("network edge list to copy\n", my_network.G.edges())  # edge list
    logger1.info('Network edge list to copy: %s', str(my_network.G.edges()))

    # print(my_network.G.edges_iter())

    generated_graph_dir = os.path.join(here, 'output', 'nx-generated.png')
    my_network.show_graph(generated_graph_dir)
    logger1.info('Generated graph saved in %s', generated_graph_dir)

    network_of_agents = mann.network_agent.NetworkAgent()
    fig_path = os.path.join(here, 'output', 'mann-generated.png')

    agent_type = {
        "network_agent_type": config.get('NetworkParameters', 'AgentType'),
        "lens_num_processing_units": config.getint(
            'LENSParameters', 'TotalNumberOfProcessingUnits'),
        "lens_agent_type": config.get('LENSParameters', 'AgentType')
    }

    network_of_agents.\
        create_multidigraph_of_agents_from_edge_list(
            n, my_network.G.edges_iter(),
            fig_path,
            agent_type=(
                config.get('NetworkParameters', 'AgentType'),
                config.getint('LENSParameters',
                              'TotalNumberOfProcessingUnits'),
                config.get('LENSParameters', 'AgentType')))

    print('print network of agents:')

    print(len(network_of_agents.G))
    for node in network_of_agents.G:
        print(node)
        print(type(node))

    print(type(network_of_agents.G))

    model_output = os.path.join(here, 'output',
                                config.get('General', 'ModelOutput'))
    # write all agent's init state (0's and None)
    network_of_agents.write_network_agent_step_info(
        -3, model_output, 'w')

    # make agents aware of predecessors
    # predecessors are agents who influence the current agent
    network_of_agents.set_predecessors_for_each_node()

    for node in network_of_agents.G:
        print(node)
        print(node.predecessors)

    # randomly select nodes from network_of_agents to seed
    num_seed = config.getint('ModelParameters', 'NumberOfAgentsToSeedOnInit')
    agents_to_seed = network_of_agents.sample_network(num_seed)
    # print("agents to seed: ", agents_to_seed)
    logger1.info('Agents seeded: %s', str(agents_to_seed))

    lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                               'UpdateFromInflInFile')

    agent_self_ex_file = here + '/' + config.get('LENSParameters',
                                                 'InflExFile')

    agent_self_out_file = here + '/' + config.get('LENSParameters',
                                                  'NewAgentStateFile')

    new_state_values_dict = {}
    types_of_inputs = {'all_neg': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
                       'all_pos': [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                       'amb_pos': [1, 0, 1, 0, 1, 0, 1, 0, 1, 0],
                       'amb_neg': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
                       'amb_good': [1, 1, 0, 0, 0, 1, 1, 0, 0, 0]}

    # seed agents who were select
    for selected_agent in agents_to_seed:
        logger1.info('Seeding agent  %s', str(selected_agent.get_key()))

        logger1.debug('Agent %s, pre-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.get_state()))

        network_of_agents.write_network_agent_step_info(
            -2, model_output, 'a')

        lens_in_writer_helper = mann.lens_in_writer.LensInWriterHelper()
        write_str = lens_in_writer_helper.generate_lens_recurrent_attitude(
            mann.helper.convert_list_to_delim_str(selected_agent.state,
                                                  delim=' '),
            mann.helper.convert_list_to_delim_str(types_of_inputs['amb_good'],
                                                  delim=' '))
        lens_in_writer_helper.write_in_file(agent_self_ex_file, write_str)

        selected_agent.call_lens(lens_in_file_dir)

        new_state_values = selected_agent.get_new_state_values_from_out_file(
            os.path.join(agent_self_out_file),
            'agent_type param is not used ... yet')
        selected_agent.temp_new_state = new_state_values[:]
        new_state_values_dict[selected_agent.agent_id] = selected_agent.\
            temp_new_state

    print("SIMULTANEOUS UPDATE SEEDS")
    for selected_agent in agents_to_seed:
        selected_agent.state = new_state_values_dict[selected_agent.agent_id]
        selected_agent.temp_new_state = None

        logger1.debug('Agent %s seeded', str(selected_agent.get_key()))

        logger1.debug('Agent %s, post-seed state: %s',
                      str(selected_agent.agent_id),
                      str(selected_agent.state))

    # agent states after seed get updated
    network_of_agents.write_network_agent_step_info(
        -1, model_output, 'a')

    logger1.info('Begin steps')

    print(network_of_agents.G.nodes())

    edge_list_file_dir = os.path.join(here, 'output', 'edge_list.gz')
    print(edge_list_file_dir)
    # network_of_agents.export_edge_list(edge_list_file_dir)
    network_of_agents.export_edge_list(edge_list_file_dir)

    for i in range(config.getint('ModelParameters', 'NumberOfTimeTicks')):
        print("STEP # ", i)
        step(i, network_of_agents, agent_type)

if __name__ == "__main__":
    main()
<commit_msg>pass different agent_type keys into network creation<commit_after>#! /usr/bin/env python

import os
import logging
import configparser
import random
import warnings

import mann.network
import mann.network_agent
import mann.helper
import mann.lens_in_writer

# from mann import network
# from mann import network_agent
# from mann import helper
# from mann import lens_in_writer

here = os.path.abspath(os.path.dirname(__file__))

# set up logging to file - see previous section for more details
logging_format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
logging.basicConfig(level=logging.DEBUG,
                    format=logging_format,
                    datefmt='%m-%d %H:%M',
                    filename=os.path.join(here, 'output', 'myapp.log'),
                    filemode='w')

# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)

formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')

# tell the handler to use this format
# set a format which is simpler for console use
console.setFormatter(formatter)

# add the handler to the root logger
logging.getLogger('').addHandler(console)

# Now, we can log to the root logger, or any other logger. First the root..
logging.info('Logger created in main()')

# Now, define a couple of other loggers which might represent areas in your
# application:

logger1 = logging.getLogger(os.path.join(here, 'myapp.area1'))
logger2 = logging.getLogger(os.path.join(here, 'myapp.area2'))

# setting up the configparser
config = configparser.ConfigParser()
config.read(os.path.join(here, 'config.ini'))

def random_select_and_update(network_of_agents):
    # randomly select nodes from network_of_agents
    # select num_update number of the nodes for update
    num_update = config.getint('ModelParameters',
                               'NumberOfAgentsToUpdatePerTimeTick')
    agents_for_update = network_of_agents.sample_network(num_update)
    print('agents for update: ', agents_for_update)
    print('key of agent for update')
    print(network_of_agents.G.nodes()[agents_for_update[0].get_key()])

    # update agents who were selected
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])

        lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                                   'UpdateFromInflInFile')

        agent_ex_file_dir = here + '/' + config.get('LENSParameters',
                                                    'AgentExFile')

        infl_ex_file_dir = here + '/' + config.get('LENSParameters',
                                                   'InflExFile')

        agent_state_out_file_dir = here + '/' + config.get('LENSParameters',
                                                           'NewAgentStateFile')

        selected_agent.update_agent_state(
            'default',
            lens_in_file=lens_in_file_dir,
            agent_ex_file=agent_ex_file_dir,
            infl_ex_file=infl_ex_file_dir,
            agent_state_out_file=agent_state_out_file_dir)

def update_simultaneous(network_of_agents, num_agents_update):
    """Simultaneously updates agents

    :param network_of_agents: NetworkX graph of agents
    :type network_of_agents: NetworkX graph

    :parm num_agents_update: Number of agents that will be picked for update
    :type num_agents_update: int

    iterates through each node in the network and calculates a new state value
    that will be saved to a temp variable.  It will then iterate though the
    network again to update each node to the temp variable
    """
    agents_for_update = network_of_agents.sample_network(num_agents_update)
    print('agents for update: ', agents_for_update)
    print('key of agent for update')
    for agent_update_key in agents_for_update:
        print(network_of_agents.G.nodes()[agent_update_key.get_key()])

    lens_in_file_dir = os.path.join(here, config.get('LENSParameters',
                                                     'UpdateFromInflInFile'))
    lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                               'UpdateFromInflInFile')

    infl_ex_file_dir = os.path.join(here, config.get('LENSParameters',
                                                     'InflExFile'))

    agent_state_out_file_dir = os.path.join(here,
                                            config.get('LENSParameters',
                                                       'NewAgentStateFile'))

    print('printing network_of_agents.G nodes')
    for node in network_of_agents.G:
        print(node)

    # save to temp state before looping to sim update
    lens_in_writer_helper = mann.lens_in_writer.LensInWriterHelper()
    for selected_agent in agents_for_update:
        print("updating: ",
              network_of_agents.G.nodes()[selected_agent.get_key()])
        print(selected_agent.temp_new_state)
        assert selected_agent.temp_new_state is None

        try:
            random_predecessor_id = random.sample(selected_agent.predecessors,
                                                  1)
        except ValueError:
            # if agent has no predecessor we skip it
            continue
        else:
            predecessor_picked = random_predecessor_id[0]

            print(type(predecessor_picked))

            print(predecessor_picked)

            write_str = lens_in_writer_helper.generate_lens_recurrent_attitude(
                mann.helper.convert_list_to_delim_str(selected_agent.state,
                                                      delim=' '),
                mann.helper.convert_list_to_delim_str(predecessor_picked.state,
                                                      delim=' '))
            lens_in_writer_helper.write_in_file(infl_ex_file_dir, write_str)

            print(lens_in_file_dir)
            selected_agent.call_lens(lens_in_file_dir)

            new_state_values = selected_agent.\
                get_new_state_values_from_out_file(
                    agent_state_out_file_dir,
                    'agent_type param is not used ... yet')
            selected_agent.temp_new_state = new_state_values

    # simultaneous update
    for selected_agent in agents_for_update:
        if len(selected_agent.predecessors) > 0:
            assert (selected_agent.temp_new_state is not None)
            selected_agent.state = selected_agent.temp_new_state
            selected_agent.temp_new_state = None
        else:
            assert len(selected_agent.predecessors) == 0
            warnings_str = "Pedecessors for Agent {}: {}".format(
                selected_agent.agent_id, selected_agent.predecessors)
            warnings.warn(warnings_str)

def step(time_tick, network_of_agents, agent_type):
    logger1.debug('STEP TIME TICK: %s', str(time_tick))

    logger1.debug('Begin random select and update network of agents')

    update_type = config.get('ModelParameters', 'UpdateType')

    if update_type == 'sequential':
        random_select_and_update(network_of_agents)
    elif update_type == 'simultaneous':
        print("Performing a simultaneous update.")
        update_simultaneous(network_of_agents,
                            config.getint(
                                'ModelParameters',
                                'NumberOfAgentsToUpdatePerTimeTick'))
    else:
        raise ValueError('Unknown simulation update type')

    network_agent_step_time_dir = os.path.join(here, 'output',
                                               'network_of_agents.pout')

    network_of_agents.write_network_agent_step_info(
        time_tick, network_agent_step_time_dir, 'a')
    logger1.debug('Time ticks %s values appended to %s',
                  str(time_tick),
                  network_agent_step_time_dir)

def main():
    logger1.info('In main.main()')
    logger1.info('Starting Mulit Agent Neural Network (MANN)')

    # creating n number of agents
    n = config.getint('NetworkParameters', 'NumberOfAgents')
    logger1.debug('Number of agents to create: %s', str(n))

    # probablity for edge creation [0, 1]
    p = config.getfloat('NetworkParameters', 'ProbEdgeCreation')
    logger1.debug('Probablity for edge creation: %s', str(p))

    # Create Erdos-Renyi graph
    my_network = mann.network.DirectedFastGNPRandomGraph(n, p)

    # print("network edge list to copy\n", my_network.G.edges())  # edge list
    logger1.info('Network edge list to copy: %s', str(my_network.G.edges()))

    # print(my_network.G.edges_iter())

    generated_graph_dir = os.path.join(here, 'output', 'nx-generated.png')
    my_network.show_graph(generated_graph_dir)
    logger1.info('Generated graph saved in %s', generated_graph_dir)

    network_of_agents = mann.network_agent.NetworkAgent()
    fig_path = os.path.join(here, 'output', 'mann-generated.png')

    agent_type = {
        "network_agent_type": config.get('NetworkParameters', 'AgentType'),
        "lens_num_processing_units": config.getint(
            'LENSParameters', 'TotalNumberOfProcessingUnits'),
        "lens_agent_type": config.get('LENSParameters', 'AgentType')
    }

    network_of_agents.\
        create_multidigraph_of_agents_from_edge_list(
            n, my_network.G.edges_iter(),
            fig_path,
            agent_type=(agent_type.get("network_agent_type"),
                        agent_type.get("lens_num_processing_units"),
                        agent_type.get("lens_agent_type"))
        )

    print('print network of agents:')

    print(len(network_of_agents.G))
    for node in network_of_agents.G:
        print(node)
        print(type(node))

    print(type(network_of_agents.G))

    model_output = os.path.join(here, 'output',
                                config.get('General', 'ModelOutput'))
    # write all agent's init state (0's and None)
    network_of_agents.write_network_agent_step_info(
        -3, model_output, 'w')

    # make agents aware of predecessors
    # predecessors are agents who influence the current agent
    network_of_agents.set_predecessors_for_each_node()

    for node in network_of_agents.G:
        print(node)
        print(node.predecessors)

    # randomly select nodes from network_of_agents to seed
    num_seed = config.getint('ModelParameters', 'NumberOfAgentsToSeedOnInit')
    agents_to_seed = network_of_agents.sample_network(num_seed)
    # print("agents to seed: ", agents_to_seed)
    logger1.info('Agents seeded: %s', str(agents_to_seed))

    lens_in_file_dir = here + '/' + config.get('LENSParameters',
                                               'UpdateFromInflInFile')

    agent_self_ex_file = here + '/' + config.get('LENSParameters',
                                                 'InflExFile')

    agent_self_out_file = here + '/' + config.get('LENSParameters',
                                                  'NewAgentStateFile')

    new_state_values_dict = {}
    types_of_inputs = {'all_neg': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
                       'all_pos': [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                       'amb_pos': [1, 0, 1, 0, 1, 0, 1, 0, 1, 0],
                       'amb_neg': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
                       'amb_good': [1, 1, 0, 0, 0, 1, 1, 0, 0, 0]}

    # seed agents who were select
    for selected_agent in agents_to_seed:
        logger1.info('Seeding agent  %s', str(selected_agent.get_key()))

        logger1.debug('Agent %s, pre-seed state: %s',
                      str(selected_agent.get_key()),
                      str(selected_agent.get_state()))

        network_of_agents.write_network_agent_step_info(
            -2, model_output, 'a')

        lens_in_writer_helper = mann.lens_in_writer.LensInWriterHelper()
        write_str = lens_in_writer_helper.generate_lens_recurrent_attitude(
            mann.helper.convert_list_to_delim_str(selected_agent.state,
                                                  delim=' '),
            mann.helper.convert_list_to_delim_str(types_of_inputs['amb_good'],
                                                  delim=' '))
        lens_in_writer_helper.write_in_file(agent_self_ex_file, write_str)

        selected_agent.call_lens(lens_in_file_dir)

        new_state_values = selected_agent.get_new_state_values_from_out_file(
            os.path.join(agent_self_out_file),
            'agent_type param is not used ... yet')
        selected_agent.temp_new_state = new_state_values[:]
        new_state_values_dict[selected_agent.agent_id] = selected_agent.\
            temp_new_state

    print("SIMULTANEOUS UPDATE SEEDS")
    for selected_agent in agents_to_seed:
        selected_agent.state = new_state_values_dict[selected_agent.agent_id]
        selected_agent.temp_new_state = None

        logger1.debug('Agent %s seeded', str(selected_agent.get_key()))

        logger1.debug('Agent %s, post-seed state: %s',
                      str(selected_agent.agent_id),
                      str(selected_agent.state))

    # agent states after seed get updated
    network_of_agents.write_network_agent_step_info(
        -1, model_output, 'a')

    logger1.info('Begin steps')

    print(network_of_agents.G.nodes())

    edge_list_file_dir = os.path.join(here, 'output', 'edge_list.gz')
    print(edge_list_file_dir)
    # network_of_agents.export_edge_list(edge_list_file_dir)
    network_of_agents.export_edge_list(edge_list_file_dir)

    for i in range(config.getint('ModelParameters', 'NumberOfTimeTicks')):
        print("STEP # ", i)
        step(i, network_of_agents, agent_type)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

__author__ = 'lundberg'

"""
For more built in configuration options see,
http://flask.pocoo.org/docs/0.10/config/#builtin-configuration-values
"""

DEBUG = False

# Database URIs
IDPROOFING_MONGO_URI = 'mongodb://'
USERDB_MONGO_URI = 'mongodb://'

# Application specific settings
LETTER_WAIT_TIME_HOURS = 336  # 2 weeks

# Celery config
CELERY_CONFIG = {
    'BROKER_URL': 'amqp://',
    'CELERY_RESULT_BACKEND': 'amqp',
    'CELERY_TASK_SERIALIZER': 'json'
}

# Secret key
SECRET_KEY = None

# Logging
LOG_FILE = None
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
LOG_LEVEL = 'INFO'
LOG_MAX_BYTES = 10000
LOG_BACKUP_COUNT = 10

EKOPOST_API_URI = 'https://api.ekopost.se'
EKOPOST_API_VERIFY_SSL = 'true'
EKOPOST_API_USER = ''
EKOPOST_API_PW = ''
EKOPOST_DEBUG_PDF = ''

# Api Spec config - https://github.com/OAI/OpenAPI-Specification
# from apispec import APISpec
# APISPEC_SPEC = APISpec(
#    title = 'eduid-idproofing-letter',
#    version = 'v1',
#    plugins = ('apispec.ext.marshmallow',),
# )

APISPEC_SPEC = None
APISPEC_SWAGGER_URL = '/swagger/'
APISPEC_SWAGGER_UI_URL = '/swagger-ui/'
<commit_msg>Upped log file size to 1 MB<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

__author__ = 'lundberg'

"""
For more built in configuration options see,
http://flask.pocoo.org/docs/0.10/config/#builtin-configuration-values
"""

DEBUG = False

# Database URIs
IDPROOFING_MONGO_URI = 'mongodb://'
USERDB_MONGO_URI = 'mongodb://'

# Application specific settings
LETTER_WAIT_TIME_HOURS = 336  # 2 weeks

# Celery config
CELERY_CONFIG = {
    'BROKER_URL': 'amqp://',
    'CELERY_RESULT_BACKEND': 'amqp',
    'CELERY_TASK_SERIALIZER': 'json'
}

# Secret key
SECRET_KEY = None

# Logging
LOG_FILE = None
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
LOG_LEVEL = 'INFO'
LOG_MAX_BYTES = 1000000  # 1 MB
LOG_BACKUP_COUNT = 10

EKOPOST_API_URI = 'https://api.ekopost.se'
EKOPOST_API_VERIFY_SSL = 'true'
EKOPOST_API_USER = ''
EKOPOST_API_PW = ''
EKOPOST_DEBUG_PDF = ''

# Api Spec config - https://github.com/OAI/OpenAPI-Specification
# from apispec import APISpec
# APISPEC_SPEC = APISpec(
#    title = 'eduid-idproofing-letter',
#    version = 'v1',
#    plugins = ('apispec.ext.marshmallow',),
# )

APISPEC_SPEC = None
APISPEC_SWAGGER_URL = '/swagger/'
APISPEC_SWAGGER_UI_URL = '/swagger-ui/'
<|endoftext|>


<commit_before>import unittest2

from aws_certificate_management import cleanup, setup_certificate

DOMAIN = '*.pro-test.wolke.is'
ACM_ARN_PREFIX = 'arn:aws:acm:eu-west-1:'
ACM_ARN_PREFIX2 = 'arn:aws:acm:eu-central-1:'

class SetupCertificateTests(unittest2.TestCase):
    def tearDown(self):
        # Cleanup might run into a race condition, where items are added
        # to the bucket just before the bucket-stack should be deleted.
        # So just retry before really giving up.
        for attempt in 1, 2, 3:
            try:
                cleanup(DOMAIN)
            except Exception:
                pass
            else:
                return

    def test_setup_certificate_called_twice(self):
        with self.assertLogs(logger='aws-certificate-management', level='INFO') as cm:
            setup_certificate(DOMAIN, 'eu-west-1')
            setup_certificate(DOMAIN, 'eu-west-1')

        logged_messages = "".join(cm.output)
        self.assertIn(ACM_ARN_PREFIX, logged_messages)

    def test_setup_certificate_different_regions(self):
        with self.assertLogs(level='INFO') as cm:
            setup_certificate(DOMAIN, 'eu-west-1')
            setup_certificate(DOMAIN, 'eu-central-1')

        logged_messages = "".join(cm.output)
        self.assertIn(ACM_ARN_PREFIX, logged_messages)
        self.assertIn(ACM_ARN_PREFIX2, logged_messages)

if __name__ == "__main__":
    unittest2.main()
<commit_msg>Rename DOMAIN to WILDCARD_DOMAIN<commit_after>import unittest2

from aws_certificate_management import cleanup, setup_certificate

WILDCARD_DOMAIN = '*.pro-test.wolke.is'
ACM_ARN_PREFIX = 'arn:aws:acm:eu-west-1:'
ACM_ARN_PREFIX2 = 'arn:aws:acm:eu-central-1:'

class SetupCertificateTests(unittest2.TestCase):
    def tearDown(self):
        # Cleanup might run into a race condition, where items are added
        # to the bucket just before the bucket-stack should be deleted.
        # So just retry before really giving up.
        for attempt in 1, 2, 3:
            try:
                cleanup(WILDCARD_DOMAIN)
            except Exception:
                pass
            else:
                return

    def test_setup_certificate_called_twice(self):
        with self.assertLogs(logger='aws-certificate-management', level='INFO') as cm:
            setup_certificate(WILDCARD_DOMAIN, 'eu-west-1')
            setup_certificate(WILDCARD_DOMAIN, 'eu-west-1')

        logged_messages = "".join(cm.output)
        self.assertIn(ACM_ARN_PREFIX, logged_messages)

    def test_setup_certificate_different_regions(self):
        with self.assertLogs(level='INFO') as cm:
            setup_certificate(WILDCARD_DOMAIN, 'eu-west-1')
            setup_certificate(WILDCARD_DOMAIN, 'eu-central-1')

        logged_messages = "".join(cm.output)
        self.assertIn(ACM_ARN_PREFIX, logged_messages)
        self.assertIn(ACM_ARN_PREFIX2, logged_messages)

if __name__ == "__main__":
    unittest2.main()
<|endoftext|>


<commit_before>from flask import current_app, redirect, url_for, request
from flask.ext import restful
from flask.ext.login import login_required, current_user, login_user, logout_user
import tweepy
from request_parsers import *
from datetime import datetime
from models import *

session = dict()

class TwitterAuth(restful.Resource):
    def get(self):
        oauth = tweepy.OAuthHandler(current_app.config['CONSUMER_KEY'], current_app.config['CONSUMER_SECRET'], secure=True)
        oauth.callback = url_for('twitter_callback', _external=True)
        auth_url = oauth.get_authorization_url()
        session['request_token'] = (oauth.request_token.key,
                    oauth.request_token.secret)
        return redirect(auth_url)

class Login(restful.Resource):
    def get(self):
        return {'status':'Welcome'}

class TwitterCallback(restful.Resource):
    def get(self):
        try:
            oauth = tweepy.OAuthHandler(current_app.config['CONSUMER_KEY'], current_app.config['CONSUMER_SECRET'], secure=True)
            token = session['request_token']
            del session['request_token']
            verifier = request.args.get('oauth_verifier')

            oauth.get_access_token(verifier)
            api = tweepy.Api(oauth.access_token())
            u = api.verify_credentails()
            if u:
                if not User.objects.get(twitter_id = u.id):
                    user = User(
                            twitter_id = u.id,
                            twitter_token = oauth.access_token,
                            twitter_secret = oauth.access_token_secret,
                            screen_name = u.screen_name,
                            registered_on = datetime.now()
                            )
                    user.save()
            user = User.objects.get(twitter_id = u.id)

            login_user(user)
            return
        except Exception as e:
            print e
            restful.abort(500, message = 'Internal Server Error.')

class MyLists(restful.Resource):
    @login_required
    def get(self):
        args = list_parser.parse_args()
        user = current_user
        try:
            pass
        except Exception as e:
            print e
            restful.abort(500, message = 'Internal Server Error.')

<commit_msg>fixes oauth<commit_after>from flask import current_app, redirect, url_for, request
from flask.ext import restful
from flask.ext.login import login_required, current_user, login_user, logout_user
import tweepy
from request_parsers import *
from datetime import datetime
from models import *

session = dict()

class TwitterAuth(restful.Resource):
    def get(self):
        oauth = tweepy.OAuthHandler(current_app.config['CONSUMER_KEY'], current_app.config['CONSUMER_SECRET'], secure=True)
        oauth.callback = url_for('twittercallback', _external=True)
        auth_url = oauth.get_authorization_url()
        session['request_token'] = (oauth.request_token.key,
                    oauth.request_token.secret)
        return redirect(auth_url)

class Login(restful.Resource):
    def get(self):
        return {'status':'Welcome'}

class TwitterCallback(restful.Resource):
    def get(self):
        try:
            oauth = tweepy.OAuthHandler(current_app.config['CONSUMER_KEY'], current_app.config['CONSUMER_SECRET'], secure=True)
            token = session['request_token']
            del session['request_token']
            verifier = request.args.get('oauth_verifier')

            oauth.get_access_token(verifier)
            api = tweepy.Api(oauth.access_token())
            u = api.verify_credentails()
            if u:
                if not User.objects.get(twitter_id = u.id):
                    user = User(
                            twitter_id = u.id,
                            twitter_token = oauth.access_token,
                            twitter_secret = oauth.access_token_secret,
                            screen_name = u.screen_name,
                            registered_on = datetime.now()
                            )
                    user.save()
            user = User.objects.get(twitter_id = u.id)

            login_user(user)
            return
        except Exception as e:
            print e
            restful.abort(500, message = 'Internal Server Error.')

class MyLists(restful.Resource):
    @login_required
    def get(self):
        args = list_parser.parse_args()
        user = current_user
        try:
            pass
        except Exception as e:
            print e
            restful.abort(500, message = 'Internal Server Error.')

<|endoftext|>


<commit_before>from tehbot.plugins import *
import time
from pony.orm import *
from datetime import datetime

class SeenPlugin(StandardCommand):
    """Shows when a user was last seen."""

    def __init__(self, db):
        StandardCommand.__init__(self, db)
        self.parser.add_argument("user", nargs=1)
        self.ircid_arg = self.parser.add_argument("-i", "--ircid")

    def commands(self):
        return ["seen", "last"]

    @db_session
    def ircid2name(self, ircid):
        settings = select(s for s in self.db.Setting if s.name == "tehbot").first()
        try:
            return settings.value["connections"][ircid]["name"]
        except:
            return ircid

    def get_action(self, message):
        arr = message.split()
        if not arr:
            return None
        verb = arr[0]
        if len(verb) < 2 or verb[-1].lower() != "s":
            return None
        verb = verb[:-1]
        v = verb.lower()

        if v[-2:] == "ie":
            verb = verb[:-2] + "y"
        elif v[-1] == "e":
            if not v.endswith("ee"):
                verb = verb[:-1]
        elif v[-2:] == "ic":
            verb = verb + "k"
        elif v[-1] not in "aeiou" and v[-2:-1] in "aeiou":
            if v[-3:-2] not in "aeiou":
                verb = verb + verb[-1]
        args = " ".join(arr[1:])
        msg = verb + "ing"
        if args:
            msg += " " + args
        return msg

    @db_session
    def execute(self, connection, event, extra):
        ircids = select(m.ircid for m in self.db.Message)[:]
        self.ircid_arg.choices = sorted(ircids)
        return StandardCommand.execute(self, connection, event, extra)

    @db_session
    def execute_parsed(self, connection, event, extra):
        user = self.pargs.user[0]
        ircid = self.pargs.ircid
        msgs = select(m for m in self.db.Message if m.nick == user and m.type == 0 and (ircid is None or m.ircid == ircid)).order_by(desc(self.db.Message.ts))[:1]
        if not msgs:
            wherestr = " on %s" % self.ircid2name(ircid) if ircid else ""
            return [("say_nolog", u"I've never seen %s%s." % (user, wherestr))]

        m = msgs[0]
        timestr = Plugin.time2str(time.time(), time.mktime(m.ts.timetuple()))
        name = self.ircid2name(m.ircid)
        action = self.get_action(m.message)
        if m.event == "action" and action is not None:
            msg = u"I saw %s %s ago on %s in %s %s." % (user, timestr, name, m.target, action)
        else:
            msg =  u"I saw %s %s ago on %s in %s saying '%s'." % (user, timestr, name, m.target, m.message)
        return [("say_nolog", msg)]

class OpHandler(ChannelJoinHandler):
    def default_settings(self):
        return { "who" : [], "whonot" : [] }

    def execute(self, connection, event, extra):
        who = event.source.nick

        if self.settings["who"] and who not in self.settings["who"]:
            return

        if self.settings["whonot"] and who in self.settings["whonot"]:
            return

        connection.mode(event.target, "+o " + who)
        print "ok"

    def values_to_add(self):
        return ChannelJoinHandler.values_to_add(self) + [ "who", "whonot" ]
<commit_msg>SeenPlugin: fix grammar<commit_after>from tehbot.plugins import *
import time
from pony.orm import *
from datetime import datetime

class SeenPlugin(StandardCommand):
    """Shows when a user was last seen."""

    def __init__(self, db):
        StandardCommand.__init__(self, db)
        self.parser.add_argument("user", nargs=1)
        self.ircid_arg = self.parser.add_argument("-i", "--ircid")

    def commands(self):
        return ["seen", "last"]

    @db_session
    def ircid2name(self, ircid):
        settings = select(s for s in self.db.Setting if s.name == "tehbot").first()
        try:
            return settings.value["connections"][ircid]["name"]
        except:
            return ircid

    def get_action(self, message):
        arr = message.split()
        if not arr:
            return None
        verb = arr[0]
        if len(verb) < 2 or verb[-1].lower() != "s":
            return None
        verb = verb[:-1]
        v = verb.lower()

        if v == "i":
            verb = "be"
        elif v[-2:] == "ie":
            verb = verb[:-2] + "y"
        elif v[-1] == "e":
            if not v.endswith("ee"):
                verb = verb[:-1]
        elif v[-2:] == "ic":
            verb = verb + "k"
        elif v[-1] not in "aeiou" and v[-2:-1] in "aeiou":
            if v[-3:-2] not in "aeiou":
                verb = verb + verb[-1]
        args = " ".join(arr[1:])
        msg = verb + "ing"
        if args:
            msg += " " + args
        return msg

    @db_session
    def execute(self, connection, event, extra):
        ircids = select(m.ircid for m in self.db.Message)[:]
        self.ircid_arg.choices = sorted(ircids)
        return StandardCommand.execute(self, connection, event, extra)

    @db_session
    def execute_parsed(self, connection, event, extra):
        user = self.pargs.user[0]
        ircid = self.pargs.ircid
        msgs = select(m for m in self.db.Message if m.nick == user and m.type == 0 and (ircid is None or m.ircid == ircid)).order_by(desc(self.db.Message.ts))[:1]
        if not msgs:
            wherestr = " on %s" % self.ircid2name(ircid) if ircid else ""
            return [("say_nolog", u"I've never seen %s%s." % (user, wherestr))]

        m = msgs[0]
        timestr = Plugin.time2str(time.time(), time.mktime(m.ts.timetuple()))
        name = self.ircid2name(m.ircid)
        action = self.get_action(m.message)
        if m.event == "action" and action is not None:
            msg = u"I saw %s %s ago on %s in %s %s." % (user, timestr, name, m.target, action)
        else:
            msg =  u"I saw %s %s ago on %s in %s saying '%s'." % (user, timestr, name, m.target, m.message)
        return [("say_nolog", msg)]

class OpHandler(ChannelJoinHandler):
    def default_settings(self):
        return { "who" : [], "whonot" : [] }

    def execute(self, connection, event, extra):
        who = event.source.nick

        if self.settings["who"] and who not in self.settings["who"]:
            return

        if self.settings["whonot"] and who in self.settings["whonot"]:
            return

        connection.mode(event.target, "+o " + who)
        print "ok"

    def values_to_add(self):
        return ChannelJoinHandler.values_to_add(self) + [ "who", "whonot" ]
<|endoftext|>


<commit_before># Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import hashlib
import os

import yaml

from tempest.common import cred_provider
from tempest import config
from tempest import exceptions
from tempest.openstack.common import lockutils
from tempest.openstack.common import log as logging

CONF = config.CONF
LOG = logging.getLogger(__name__)

def read_accounts_yaml(path):
    yaml_file = open(path, 'r')
    accounts = yaml.load(yaml_file)
    return accounts

class Accounts(cred_provider.CredentialProvider):

    def __init__(self, name):
        super(Accounts, self).__init__(name)
        if os.path.isfile(CONF.auth.test_accounts_file):
            accounts = read_accounts_yaml(CONF.auth.test_accounts_file)
            self.use_default_creds = False
        else:
            accounts = {}
            self.use_default_creds = True
        self.hash_dict = self.get_hash_dict(accounts)
        self.accounts_dir = os.path.join(CONF.lock_path, 'test_accounts')
        self.isolated_creds = {}

    @classmethod
    def get_hash_dict(cls, accounts):
        hash_dict = {}
        for account in accounts:
            temp_hash = hashlib.md5()
            temp_hash.update(str(account))
            hash_dict[temp_hash.hexdigest()] = account
        return hash_dict

    def is_multi_user(self):
        # Default credentials is not a valid option with locking Account
        if self.use_default_creds:
            raise exceptions.InvalidConfiguration(
                "Account file %s doesn't exist" % CONF.auth.test_accounts_file)
        else:
            return len(self.hash_dict) > 1

    def is_multi_tenant(self):
        return self.is_multi_user()

    def _create_hash_file(self, hash_string):
        path = os.path.join(os.path.join(self.accounts_dir, hash_string))
        if not os.path.isfile(path):
            open(path, 'w').close()
            return True
        return False

    @lockutils.synchronized('test_accounts_io', external=True)
    def _get_free_hash(self, hashes):
        if not os.path.isdir(self.accounts_dir):
            os.mkdir(self.accounts_dir)
            # Create File from first hash (since none are in use)
            self._create_hash_file(hashes[0])
            return hashes[0]
        for _hash in hashes:
            res = self._create_hash_file(_hash)
            if res:
                return _hash
        msg = 'Insufficient number of users provided'
        raise exceptions.InvalidConfiguration(msg)

    def _get_creds(self):
        if self.use_default_creds:
            raise exceptions.InvalidConfiguration(
                "Account file %s doesn't exist" % CONF.auth.test_accounts_file)
        free_hash = self._get_free_hash(self.hash_dict.keys())
        return self.hash_dict[free_hash]

    @lockutils.synchronized('test_accounts_io', external=True)
    def remove_hash(self, hash_string):
        hash_path = os.path.join(self.accounts_dir, hash_string)
        if not os.path.isfile(hash_path):
            LOG.warning('Expected an account lock file %s to remove, but '
                        'one did not exist')
        else:
            os.remove(hash_path)
            if not os.listdir(self.accounts_dir):
                os.rmdir(self.accounts_dir)

    def get_hash(self, creds):
        for _hash in self.hash_dict:
            # Comparing on the attributes that were read from the YAML
            if all([getattr(creds, k) == self.hash_dict[_hash][k] for k in
                    creds.get_init_attributes()]):
                return _hash
        raise AttributeError('Invalid credentials %s' % creds)

    def remove_credentials(self, creds):
        _hash = self.get_hash(creds)
        self.remove_hash(_hash)

    def get_primary_creds(self):
        if self.isolated_creds.get('primary'):
            return self.isolated_creds.get('primary')
        creds = self._get_creds()
        primary_credential = cred_provider.get_credentials(**creds)
        self.isolated_creds['primary'] = primary_credential
        return primary_credential

    def get_alt_creds(self):
        if self.isolated_creds.get('alt'):
            return self.isolated_creds.get('alt')
        creds = self._get_creds()
        alt_credential = cred_provider.get_credentials(**creds)
        self.isolated_creds['alt'] = alt_credential
        return alt_credential

    def clear_isolated_creds(self):
        for creds in self.isolated_creds.values():
            self.remove_credentials(creds)

    def get_admin_creds(self):
        msg = ('If admin credentials are available tenant_isolation should be'
               ' used instead')
        raise NotImplementedError(msg)

class NotLockingAccounts(Accounts):
    """Credentials provider which always returns the first and second
    configured accounts as primary and alt users.
    This credential provider can be used in case of serial test execution
    to preserve the current behaviour of the serial tempest run.
    """

    def _unique_creds(self, cred_arg=None):
        """Verify that the configured credentials are valid and distinct """
        if self.use_default_creds:
            try:
                user = self.get_primary_creds()
                alt_user = self.get_alt_creds()
                return getattr(user, cred_arg) != getattr(alt_user, cred_arg)
            except exceptions.InvalidCredentials as ic:
                msg = "At least one of the configured credentials is " \
                      "not valid: %s" % ic.message
                raise exceptions.InvalidConfiguration(msg)
        else:
            # TODO(andreaf) Add a uniqueness check here
            return len(self.hash_dict) > 1

    def is_multi_user(self):
        return self._unique_creds('username')

    def is_multi_tenant(self):
        return self._unique_creds('tenant_id')

    def get_creds(self, id):
        try:
            # No need to sort the dict as within the same python process
            # the HASH seed won't change, so subsequent calls to keys()
            # will return the same result
            _hash = self.hash_dict.keys()[id]
        except IndexError:
            msg = 'Insufficient number of users provided'
            raise exceptions.InvalidConfiguration(msg)
        return self.hash_dict[_hash]

    def get_primary_creds(self):
        if self.isolated_creds.get('primary'):
            return self.isolated_creds.get('primary')
        if not self.use_default_creds:
            creds = self.get_creds(0)
            primary_credential = cred_provider.get_credentials(**creds)
        else:
            primary_credential = cred_provider.get_configured_credentials(
                'user')
        self.isolated_creds['primary'] = primary_credential
        return primary_credential

    def get_alt_creds(self):
        if self.isolated_creds.get('alt'):
            return self.isolated_creds.get('alt')
        if not self.use_default_creds:
            creds = self.get_creds(1)
            alt_credential = cred_provider.get_credentials(**creds)
        else:
            alt_credential = cred_provider.get_configured_credentials(
                'alt_user')
        self.isolated_creds['alt'] = alt_credential
        return alt_credential

    def clear_isolated_creds(self):
        self.isolated_creds = {}

    def get_admin_creds(self):
        return cred_provider.get_configured_credentials(
            "identity_admin", fill_in=False)
<commit_msg>Fix missing variable in test accounts warning log<commit_after># Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import hashlib
import os

import yaml

from tempest.common import cred_provider
from tempest import config
from tempest import exceptions
from tempest.openstack.common import lockutils
from tempest.openstack.common import log as logging

CONF = config.CONF
LOG = logging.getLogger(__name__)

def read_accounts_yaml(path):
    yaml_file = open(path, 'r')
    accounts = yaml.load(yaml_file)
    return accounts

class Accounts(cred_provider.CredentialProvider):

    def __init__(self, name):
        super(Accounts, self).__init__(name)
        if os.path.isfile(CONF.auth.test_accounts_file):
            accounts = read_accounts_yaml(CONF.auth.test_accounts_file)
            self.use_default_creds = False
        else:
            accounts = {}
            self.use_default_creds = True
        self.hash_dict = self.get_hash_dict(accounts)
        self.accounts_dir = os.path.join(CONF.lock_path, 'test_accounts')
        self.isolated_creds = {}

    @classmethod
    def get_hash_dict(cls, accounts):
        hash_dict = {}
        for account in accounts:
            temp_hash = hashlib.md5()
            temp_hash.update(str(account))
            hash_dict[temp_hash.hexdigest()] = account
        return hash_dict

    def is_multi_user(self):
        # Default credentials is not a valid option with locking Account
        if self.use_default_creds:
            raise exceptions.InvalidConfiguration(
                "Account file %s doesn't exist" % CONF.auth.test_accounts_file)
        else:
            return len(self.hash_dict) > 1

    def is_multi_tenant(self):
        return self.is_multi_user()

    def _create_hash_file(self, hash_string):
        path = os.path.join(os.path.join(self.accounts_dir, hash_string))
        if not os.path.isfile(path):
            open(path, 'w').close()
            return True
        return False

    @lockutils.synchronized('test_accounts_io', external=True)
    def _get_free_hash(self, hashes):
        if not os.path.isdir(self.accounts_dir):
            os.mkdir(self.accounts_dir)
            # Create File from first hash (since none are in use)
            self._create_hash_file(hashes[0])
            return hashes[0]
        for _hash in hashes:
            res = self._create_hash_file(_hash)
            if res:
                return _hash
        msg = 'Insufficient number of users provided'
        raise exceptions.InvalidConfiguration(msg)

    def _get_creds(self):
        if self.use_default_creds:
            raise exceptions.InvalidConfiguration(
                "Account file %s doesn't exist" % CONF.auth.test_accounts_file)
        free_hash = self._get_free_hash(self.hash_dict.keys())
        return self.hash_dict[free_hash]

    @lockutils.synchronized('test_accounts_io', external=True)
    def remove_hash(self, hash_string):
        hash_path = os.path.join(self.accounts_dir, hash_string)
        if not os.path.isfile(hash_path):
            LOG.warning('Expected an account lock file %s to remove, but '
                        'one did not exist' % hash_path)
        else:
            os.remove(hash_path)
            if not os.listdir(self.accounts_dir):
                os.rmdir(self.accounts_dir)

    def get_hash(self, creds):
        for _hash in self.hash_dict:
            # Comparing on the attributes that were read from the YAML
            if all([getattr(creds, k) == self.hash_dict[_hash][k] for k in
                    creds.get_init_attributes()]):
                return _hash
        raise AttributeError('Invalid credentials %s' % creds)

    def remove_credentials(self, creds):
        _hash = self.get_hash(creds)
        self.remove_hash(_hash)

    def get_primary_creds(self):
        if self.isolated_creds.get('primary'):
            return self.isolated_creds.get('primary')
        creds = self._get_creds()
        primary_credential = cred_provider.get_credentials(**creds)
        self.isolated_creds['primary'] = primary_credential
        return primary_credential

    def get_alt_creds(self):
        if self.isolated_creds.get('alt'):
            return self.isolated_creds.get('alt')
        creds = self._get_creds()
        alt_credential = cred_provider.get_credentials(**creds)
        self.isolated_creds['alt'] = alt_credential
        return alt_credential

    def clear_isolated_creds(self):
        for creds in self.isolated_creds.values():
            self.remove_credentials(creds)

    def get_admin_creds(self):
        msg = ('If admin credentials are available tenant_isolation should be'
               ' used instead')
        raise NotImplementedError(msg)

class NotLockingAccounts(Accounts):
    """Credentials provider which always returns the first and second
    configured accounts as primary and alt users.
    This credential provider can be used in case of serial test execution
    to preserve the current behaviour of the serial tempest run.
    """

    def _unique_creds(self, cred_arg=None):
        """Verify that the configured credentials are valid and distinct """
        if self.use_default_creds:
            try:
                user = self.get_primary_creds()
                alt_user = self.get_alt_creds()
                return getattr(user, cred_arg) != getattr(alt_user, cred_arg)
            except exceptions.InvalidCredentials as ic:
                msg = "At least one of the configured credentials is " \
                      "not valid: %s" % ic.message
                raise exceptions.InvalidConfiguration(msg)
        else:
            # TODO(andreaf) Add a uniqueness check here
            return len(self.hash_dict) > 1

    def is_multi_user(self):
        return self._unique_creds('username')

    def is_multi_tenant(self):
        return self._unique_creds('tenant_id')

    def get_creds(self, id):
        try:
            # No need to sort the dict as within the same python process
            # the HASH seed won't change, so subsequent calls to keys()
            # will return the same result
            _hash = self.hash_dict.keys()[id]
        except IndexError:
            msg = 'Insufficient number of users provided'
            raise exceptions.InvalidConfiguration(msg)
        return self.hash_dict[_hash]

    def get_primary_creds(self):
        if self.isolated_creds.get('primary'):
            return self.isolated_creds.get('primary')
        if not self.use_default_creds:
            creds = self.get_creds(0)
            primary_credential = cred_provider.get_credentials(**creds)
        else:
            primary_credential = cred_provider.get_configured_credentials(
                'user')
        self.isolated_creds['primary'] = primary_credential
        return primary_credential

    def get_alt_creds(self):
        if self.isolated_creds.get('alt'):
            return self.isolated_creds.get('alt')
        if not self.use_default_creds:
            creds = self.get_creds(1)
            alt_credential = cred_provider.get_credentials(**creds)
        else:
            alt_credential = cred_provider.get_configured_credentials(
                'alt_user')
        self.isolated_creds['alt'] = alt_credential
        return alt_credential

    def clear_isolated_creds(self):
        self.isolated_creds = {}

    def get_admin_creds(self):
        return cred_provider.get_configured_credentials(
            "identity_admin", fill_in=False)
<|endoftext|>


<commit_before>import datetime

from django.test import TestCase, Client

from pdl.models import Proyecto
from stats.models import Dispensed

class TestStatsViews(TestCase):
    def setUp(self):
        Proyecto.objects.create(**{
            'numero_proyecto': '02764',
            'time_created': datetime.datetime.now(),
            'fecha_presentacion': '2010-10-10',
        })
        Dispensed.objects.create(**{
            'total_approved': 1444,
            'total_dispensed': 864,
            'dispensed_by_plenary': 23,
            'dispensed_by_spokesmen': 12,
            'dispensed_others': 11,
        })

    def test_index(self):
        c = Client()
        response = c.get('/stats/')
        self.assertEqual(200, response.status_code)
<commit_msg>added comisioncount item to test stats<commit_after>import datetime

from django.test import TestCase, Client

from pdl.models import Proyecto
from stats.models import Dispensed
from stats.models import ComisionCount

class TestStatsViews(TestCase):
    def setUp(self):
        Proyecto.objects.create(**{
            'numero_proyecto': '02764',
            'time_created': datetime.datetime.now(),
            'fecha_presentacion': '2010-10-10',
        })
        Dispensed.objects.create(**{
            'total_approved': 1444,
            'total_dispensed': 864,
            'dispensed_by_plenary': 23,
            'dispensed_by_spokesmen': 12,
            'dispensed_others': 11,
        })
        ComisionCount.objects.create(**{
            'count': 7,
            'comision': 'Justicia',
        })

    def test_index(self):
        c = Client()
        response = c.get('/stats/')
        self.assertEqual(200, response.status_code)
<|endoftext|>


<commit_before>import requests
import boto # AWS python SDK
from boto.s3.key import Key

# CONFIG
url = 'http://localhost/portfolio/' # locan endpoint
bucket_name = 'lisa.codes' # name of the bucket we want to use

S3 = boto.connect_s3()
bucket = S3.get_bucket(bucket_name)

html = [
	'index.php',
	'404.php',
	'project/bandr/index.php',
	'project/booksatwork/index.php',
	'project/inctrl/index.php',
	'project/infographics/index.php',
	'project/lefkowitz/index.php',
	'project/parentalcontrols/index.php',
	'project/platform/index.php'
]

for slug in html:

	r = requests.get( url + slug + '/?deploy=true' )

	print 'deploying ' + slug

	k = Key(bucket)
	k.key = slug.replace('.php', '.html')
	k.content_type = 'text/html'
	k.set_contents_from_string(r.content)

css = [
	'css/style.css'
]

for slug in css:

	r = requests.get( url + slug )

	print 'deploying ' + slug

	k = Key(bucket)
	k.key = slug
	k.content_type = 'text/css'
	k.set_contents_from_string(r.content)<commit_msg>Deleted deploy script to avoid horrors<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
import argparse
import subprocess
import yaml
import os

def last_git_modified(path, n=1):
    return subprocess.check_output([
        'git',
        'log',
        '-n', str(n),
        '--pretty=format:%h',
        path
    ]).decode('utf-8').split('\n')[-1]

def build_user_image(image_name, commit_range=None, push=False):
    if commit_range:
        image_touched = subprocess.check_output([
            'git', 'diff', '--name-only', commit_range, 'user-image',
        ]).decode('utf-8').strip() != ''
        if not image_touched:
            print("user-image not touched, not building")
            return

    # Pull last version of image to maximize cache use
    last_image_tag = last_git_modified('user-image', 2)
    last_image_spec = image_name + ':' + last_image_tag
    try:
        subprocess.check_call([
            'docker', 'pull', last_image_spec
        ])
    except subprocess.CalledProcessError:
        pass

    tag = last_git_modified('user-image')
    image_spec = image_name + ':' + tag

    subprocess.check_call([
        'docker', 'build', '--cache-from', last_image_spec, '-t', image_spec, 'user-image'
    ])
    if push:
        subprocess.check_call([
            'docker', 'push', image_spec
        ])
    print('build completed for image', image_spec)

def deploy(release, install):
    # Set up helm!
    subprocess.check_call(['helm', 'repo', 'update'])

    singleuser_tag = last_git_modified('user-image')

    with open('datahub/config.yaml') as f:
        config = yaml.safe_load(f)

    if install:
        helm = [
            'helm', 'install',
            '--name', release,
            '--namespace', release,
            'jupyterhub/jupyterhub',
            '--version', config['version'],
            '-f', 'datahub/config.yaml',
            '-f', os.path.join('datahub', 'secrets', release + '.yaml'),
            '--set', 'singleuser.image.tag={}'.format(singleuser_tag)
        ]
    else:
        helm = [
            'helm', 'upgrade', release,
            'jupyterhub/jupyterhub',
            '--version', config['version'],
            '-f', 'datahub/config.yaml',
            '-f', os.path.join('datahub', 'secrets', release + '.yaml'),
            '--set', 'singleuser.image.tag={}'.format(singleuser_tag)
        ]

    subprocess.check_call(helm)

def main():
    argparser = argparse.ArgumentParser()
    argparser.add_argument(
        '--user-image-spec',
        default='berkeleydsep/datahub-user'
    )
    subparsers = argparser.add_subparsers(dest='action')

    build_parser = subparsers.add_parser('build', description='Build & Push images')
    build_parser.add_argument('--commit-range', help='Range of commits to consider when building images')
    build_parser.add_argument('--push', action='store_true')

    deploy_parser = subparsers.add_parser('deploy', description='Deploy with helm')
    deploy_parser.add_argument('release', default='prod')
    deploy_parser.add_argument('--install', action='store_true')

    args = argparser.parse_args()

    if args.action == 'build':
        build_user_image(args.user_image_spec, args.commit_range, args.push)
    else:
        deploy(args.release, args.install)

main()
<commit_msg>Keep trying older commits until we find an image that works<commit_after>#!/usr/bin/env python3
import argparse
import subprocess
import yaml
import os

def last_git_modified(path, n=1):
    return subprocess.check_output([
        'git',
        'log',
        '-n', str(n),
        '--pretty=format:%h',
        path
    ]).decode('utf-8').split('\n')[-1]

def build_user_image(image_name, commit_range=None, push=False):
    if commit_range:
        image_touched = subprocess.check_output([
            'git', 'diff', '--name-only', commit_range, 'user-image',
        ]).decode('utf-8').strip() != ''
        if not image_touched:
            print("user-image not touched, not building")
            return

    # Pull last available version of image to maximize cache use
    try_count = 0
    while try_count < 50:
        last_image_tag = last_git_modified('user-image', try_count + 2)
        last_image_spec = image_name + ':' + last_image_tag
        try:
            subprocess.check_call([
                'docker', 'pull', last_image_spec
            ])
            break
        except subprocess.CalledProcessError:
            try_count += 1
            pass

    tag = last_git_modified('user-image')
    image_spec = image_name + ':' + tag

    subprocess.check_call([
        'docker', 'build', '--cache-from', last_image_spec, '-t', image_spec, 'user-image'
    ])
    if push:
        subprocess.check_call([
            'docker', 'push', image_spec
        ])
    print('build completed for image', image_spec)

def deploy(release, install):
    # Set up helm!
    subprocess.check_call(['helm', 'repo', 'update'])

    singleuser_tag = last_git_modified('user-image')

    with open('datahub/config.yaml') as f:
        config = yaml.safe_load(f)

    if install:
        helm = [
            'helm', 'install',
            '--name', release,
            '--namespace', release,
            'jupyterhub/jupyterhub',
            '--version', config['version'],
            '-f', 'datahub/config.yaml',
            '-f', os.path.join('datahub', 'secrets', release + '.yaml'),
            '--set', 'singleuser.image.tag={}'.format(singleuser_tag)
        ]
    else:
        helm = [
            'helm', 'upgrade', release,
            'jupyterhub/jupyterhub',
            '--version', config['version'],
            '-f', 'datahub/config.yaml',
            '-f', os.path.join('datahub', 'secrets', release + '.yaml'),
            '--set', 'singleuser.image.tag={}'.format(singleuser_tag)
        ]

    subprocess.check_call(helm)

def main():
    argparser = argparse.ArgumentParser()
    argparser.add_argument(
        '--user-image-spec',
        default='berkeleydsep/datahub-user'
    )
    subparsers = argparser.add_subparsers(dest='action')

    build_parser = subparsers.add_parser('build', description='Build & Push images')
    build_parser.add_argument('--commit-range', help='Range of commits to consider when building images')
    build_parser.add_argument('--push', action='store_true')

    deploy_parser = subparsers.add_parser('deploy', description='Deploy with helm')
    deploy_parser.add_argument('release', default='prod')
    deploy_parser.add_argument('--install', action='store_true')

    args = argparser.parse_args()

    if args.action == 'build':
        build_user_image(args.user_image_spec, args.commit_range, args.push)
    else:
        deploy(args.release, args.install)

main()
<|endoftext|>


<commit_before>__version__ = '0.6'

class TemplatesAdminException(Exception):
    pass
<commit_msg>Increment the version<commit_after>__version__ = '0.7'

class TemplatesAdminException(Exception):
    pass
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides utilities that may be especially useful to plugins."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import bleach

# pylint: disable=g-bad-import-order
# Google-only: import markdown_freewisdom
import markdown
import six

from tensorboard.backend import experiment_id as _experiment_id

_ALLOWED_ATTRIBUTES = {
    "a": ["href", "title"],
    "img": ["src", "title", "alt"],
}

_ALLOWED_TAGS = [
    "ul",
    "ol",
    "li",
    "p",
    "pre",
    "code",
    "blockquote",
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
    "hr",
    "br",
    "strong",
    "em",
    "a",
    "img",
    "table",
    "thead",
    "tbody",
    "td",
    "tr",
    "th",
]

# Cache Markdown converter to avoid expensive initialization at each
# call to `markdown_to_safe_html`.
_markdown = markdown.Markdown(extensions=["markdown.extensions.tables"])

def markdown_to_safe_html(markdown_string):
    """Convert Markdown to HTML that's safe to splice into the DOM.

    Arguments:
      markdown_string: A Unicode string or UTF-8--encoded bytestring
        containing Markdown source. Markdown tables are supported.

    Returns:
      A string containing safe HTML.
    """
    warning = ""
    # Convert to utf-8 whenever we have a binary input.
    if isinstance(markdown_string, six.binary_type):
        markdown_string_decoded = markdown_string.decode("utf-8")
        # Remove null bytes and warn if there were any, since it probably means
        # we were given a bad encoding.
        markdown_string = markdown_string_decoded.replace(u"\x00", u"")
        num_null_bytes = len(markdown_string_decoded) - len(markdown_string)
        if num_null_bytes:
            warning = (
                "<!-- WARNING: discarded %d null bytes in markdown string "
                "after UTF-8 decoding -->\n"
            ) % num_null_bytes

    string_html = _markdown.convert(markdown_string)
    string_sanitized = bleach.clean(
        string_html, tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRIBUTES
    )
    return warning + string_sanitized

def experiment_id(environ):
    """Determine the experiment ID associated with a WSGI request.

    Each request to TensorBoard has an associated experiment ID, which is
    always a string and may be empty. This experiment ID should be passed
    to data providers.

    Args:
      environ: A WSGI environment `dict`. For a Werkzeug request, this is
        `request.environ`.

    Returns:
      A experiment ID, as a possibly-empty `str`.
    """
    return environ.get(_experiment_id.WSGI_ENVIRON_KEY, "")
<commit_msg>plugin_util: use thread-local Markdown converter (#3491)<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides utilities that may be especially useful to plugins."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import threading

import bleach

# pylint: disable=g-bad-import-order
# Google-only: import markdown_freewisdom
import markdown
import six

from tensorboard.backend import experiment_id as _experiment_id

_ALLOWED_ATTRIBUTES = {
    "a": ["href", "title"],
    "img": ["src", "title", "alt"],
}

_ALLOWED_TAGS = [
    "ul",
    "ol",
    "li",
    "p",
    "pre",
    "code",
    "blockquote",
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
    "hr",
    "br",
    "strong",
    "em",
    "a",
    "img",
    "table",
    "thead",
    "tbody",
    "td",
    "tr",
    "th",
]

# Cache Markdown converter to avoid expensive initialization at each
# call to `markdown_to_safe_html`.
class _MarkdownStore(threading.local):
    def __init__(self):
        self.markdown = markdown.Markdown(
            extensions=["markdown.extensions.tables"]
        )

_MARKDOWN_STORE = _MarkdownStore()

def markdown_to_safe_html(markdown_string):
    """Convert Markdown to HTML that's safe to splice into the DOM.

    Arguments:
      markdown_string: A Unicode string or UTF-8--encoded bytestring
        containing Markdown source. Markdown tables are supported.

    Returns:
      A string containing safe HTML.
    """
    warning = ""
    # Convert to utf-8 whenever we have a binary input.
    if isinstance(markdown_string, six.binary_type):
        markdown_string_decoded = markdown_string.decode("utf-8")
        # Remove null bytes and warn if there were any, since it probably means
        # we were given a bad encoding.
        markdown_string = markdown_string_decoded.replace(u"\x00", u"")
        num_null_bytes = len(markdown_string_decoded) - len(markdown_string)
        if num_null_bytes:
            warning = (
                "<!-- WARNING: discarded %d null bytes in markdown string "
                "after UTF-8 decoding -->\n"
            ) % num_null_bytes

    string_html = _MARKDOWN_STORE.markdown.convert(markdown_string)
    string_sanitized = bleach.clean(
        string_html, tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRIBUTES
    )
    return warning + string_sanitized

def experiment_id(environ):
    """Determine the experiment ID associated with a WSGI request.

    Each request to TensorBoard has an associated experiment ID, which is
    always a string and may be empty. This experiment ID should be passed
    to data providers.

    Args:
      environ: A WSGI environment `dict`. For a Werkzeug request, this is
        `request.environ`.

    Returns:
      A experiment ID, as a possibly-empty `str`.
    """
    return environ.get(_experiment_id.WSGI_ENVIRON_KEY, "")
<|endoftext|>


<commit_before>
import numpy as np
import tensorflow as tf
from tensorlayer.layers import Layer
from tensorlayer import logging

class Model():
    """The :class:`Model` class represents a neural network.

    Parameters
    -----------
    inputs : a Layer or list of Layer
        The input(s) to the model.
    outputs : a Layer or list of Layer
        The output(s) to the model.
    name : None or str
        The name of the model.
    """

    @property
    def inputs(self):
        return self._inputs

    @property
    def outputs(self):
        return self._outputs

    def __init__(self, inputs, outputs, name):
        '''

        :param inputs: Layer or list of Layer
        :param outputs: Layer or list of Layer
        :param name: str
        '''
        # Model properties
        self.name = name

        # check type of inputs and outputs
        check_order = ['inputs', 'outputs']
        for co, check_argu in enumerate([inputs, outputs]):
            if isinstance(check_argu, Layer):
                pass
            elif isinstance(check_argu, list):
                for idx in range(len(check_argu)):
                    if not isinstance(check_argu[idx], Layer):
                        raise TypeError(
                            "The argument %s should be either Layer or a list of Layer "
                            % (check_order[co]) +
                            "but the %s[%d] is detected as %s"
                            % (check_order[co], idx, type(check_argu[idx]))
                        )
            else:
                raise TypeError("The argument %s should be either Layer or a list of Layer but received %s" %
                                (check_order[co], type(check_argu)))

        # Model inputs and outputs
        self._inputs = inputs if isinstance(inputs, list) else [inputs]
        self._outputs = outputs

        # weights

        self._weights = list()
        outputs_list = self._outputs if isinstance(self._outputs, list) else [self._outputs]
        for out in outputs_list:
            current = out
            while current is not None:
                if current.weights is not None:
                    self._weights.extend(current.weights)
                # FIXME: assume each layer has only one prev layer
                current = current._input_layer

        # Model state: train or test
        self.is_train = None

    def __call__(self, inputs, is_train=None):
        """

        :param inputs: Tensor or list of Tensor, numpy.ndarray of list of numpy.ndarray (if in eager mode)
        :param is_train: boolean
        :return:
        """

        if is_train is None and self.is_train is None:
            raise ValueError("Training / inference mode not defined. Argument `is_train` should be set as True / False. Otherwise please use `Model.train()` / `Model.eval()` to switch the mode.")
        elif is_train is not None and self.is_train is not None:
            if is_train == self.is_train:
                logging.warning("Training / inference mode redefined redundantly. Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode.")
            else:
                raise AttributeError("Training / inference mode mismatch. The argument `is_train` is set as %s, " % is_train +
                                     "but the mode is currently set as %s. " % ('Training by Model.train()' if self.is_train else 'Inference by Model.eval()') +
                                     "Please EITHER use the argument `is_train` OR `Model.train()` / `Model.test()` to define the mode.")

        # convert inputs to tensor if it is originally not
        if isinstance(inputs, list):
            for idx in range(len(inputs)):
                inputs[idx] = tf.convert_to_tensor(inputs[idx])
        else:
            inputs = tf.convert_to_tensor(inputs)

        # convert inputs to list for convenience
        inputs_list = inputs if isinstance(inputs, list) else [inputs]
        outputs_list = self._outputs if isinstance(self._outputs, list) else [self._outputs]
        results = list()
        memory = dict()

        for out in outputs_list:
            stacked_layers = list()
            current = out
            while current is not None:
                stacked_layers.append(current)
                # FIXME: assume each layer has only one prev layer
                current = current._input_layer

            idx_of_input = self._find_idx_of_inputs(stacked_layers[-1])
            z = inputs_list[idx_of_input]

            for layer in stacked_layers[::-1]:
                if layer.name in memory:
                    z = memory[layer.name]
                else:
                    # FIXME: not sure if there is a better way
                    if is_train is not None:
                        layer.is_train = is_train
                    else:
                        layer.is_train = self.is_train
                    # FIXME: assume each layer has only one prev layer
                    # z = layer.forward(z)
                    z = layer(z)
                    memory[layer.name] = z
            results.append(z.outputs)

        if not isinstance(self._outputs, list):
            return results[0]
        else:
            return results

    @property
    def weights(self):
        return self._weights

    def train(self):
        self.is_train = True

    def eval(self):
        self.is_train = False

    def test(self):
        self.eval()

    def infer(self):
        self.eval()

    def _find_idx_of_inputs(self, target_input):
        """
        Return the index of the target_input in self._inputs.
        Return -1 if not found.

        :param target_input: the input layer needs to be located
        :return:
        """
        if isinstance(self._inputs, list):
            for idx, input in enumerate(self._inputs):
                if input is target_input:
                    return idx
        return -1

    def __str__(self):
        return "  {} ({}) outputs_shape: {}".format(
            self.__class__.__name__, self.name, [tuple(['batch_size'] + o._outputs_shape[1:]) for o in self.outputs]
        )  #_outputs_shape)#outputs.get_shape().as_list())

    def print_all_layers(self):
        for out in self._outputs:
            stacked_layers = list()
            current = out
            while current is not None:
                print(current.name, current == self._inputs)
                stacked_layers.append(current)
                current = current._input_layer
        pass

    ## raise Exceptions for old version codes
    def count_params(self, **kwargs):
        raise Exception("please change count_params --> count_weights")

    def print_params(self, **kwargs):
        raise Exception("please change print_params --> print_weights")

    @property
    def all_params(self):
        raise Exception("please change all_params --> weights")

    @property
    def all_drop(self):
        raise Exception("all_drop is deprecated")
<commit_msg>update, now the model core supports dynamic network<commit_after>
import numpy as np
import tensorflow as tf
from tensorlayer.layers import Layer
from tensorlayer import logging

class Model():
    """The :class:`Model` class represents a neural network.

    Parameters
    -----------
    inputs : a Layer or list of Layer
        The input(s) to the model.
    outputs : a Layer or list of Layer
        The output(s) to the model.
    name : None or str
        The name of the model.
    """

    @property
    def inputs(self):
        return self._inputs

    @property
    def outputs(self):
        return self._outputs

    def __init__(self, inputs=None, outputs=None, name=None):
        '''

        :param inputs: Layer or list of Layer
        :param outputs: Layer or list of Layer
        :param name: str
        '''
        # Model properties
        # TODO: model auto naming
        self.name = name

        # Model state: train or test
        self.is_train = None

        # Model weights
        self._weights = None

        if inputs is None and outputs is None:
            pass

        else:
            # check type of inputs and outputs
            check_order = ['inputs', 'outputs']
            for co, check_argu in enumerate([inputs, outputs]):
                if isinstance(check_argu, Layer):
                    pass
                elif isinstance(check_argu, list):
                    for idx in range(len(check_argu)):
                        if not isinstance(check_argu[idx], Layer):
                            raise TypeError(
                                "The argument %s should be either Layer or a list of Layer "
                                % (check_order[co]) +
                                "but the %s[%d] is detected as %s"
                                % (check_order[co], idx, type(check_argu[idx]))
                            )
                else:
                    raise TypeError("The argument %s should be either Layer or a list of Layer but received %s" %
                                    (check_order[co], type(check_argu)))

        # Model inputs and outputs
        self._inputs = inputs
        self._outputs = outputs

    def __call__(self, inputs, is_train=None, **kwargs):
        """

        :param inputs: Tensor or list of Tensor, numpy.ndarray of list of numpy.ndarray (if in eager mode)
        :param is_train: boolean
        :return:
        """

        if is_train is None and self.is_train is None:
            raise ValueError("Training / inference mode not defined. Argument `is_train` should be set as True / False. Otherwise please use `Model.train()` / `Model.eval()` to switch the mode.")
        elif is_train is not None and self.is_train is not None:
            if is_train == self.is_train:
                logging.warning("Training / inference mode redefined redundantly. Please EITHER use the argument `is_train` OR `Model.train()` / `Model.eval()` to define the mode.")
            else:
                raise AttributeError("Training / inference mode mismatch. The argument `is_train` is set as %s, " % is_train +
                                     "but the mode is currently set as %s. " % ('Training by Model.train()' if self.is_train else 'Inference by Model.eval()') +
                                     "Please EITHER use the argument `is_train` OR `Model.train()` / `Model.test()` to define the mode.")

        # convert inputs to tensor if it is originally not
        if isinstance(inputs, list):
            for idx in range(len(inputs)):
                inputs[idx] = tf.convert_to_tensor(inputs[idx])
        else:
            inputs = tf.convert_to_tensor(inputs)

        # FIXME: currently using self._outputs to judge static network or dynamic network
        if self._outputs is not None:
            # self._inputs and self._outputs are defined when the model is created

            # convert inputs to list for convenience
            # inputs_list = inputs if isinstance(inputs, list) else [inputs]
            outputs_list = self._outputs if isinstance(self._outputs, list) else [self._outputs]
            results = list()
            memory = dict()

            for out in outputs_list:
                stacked_layers = list()
                current = out
                while current is not None:
                    stacked_layers.append(current)
                    # FIXME: assume each layer has only one prev layer
                    current = current._input_layer

                if isinstance(self.inputs, list):
                    idx_of_input = self._find_idx_of_inputs(stacked_layers[-1])
                    z = inputs[idx_of_input]
                else:
                    z = inputs

                for layer in stacked_layers[::-1]:
                    if layer.name in memory:
                        z = memory[layer.name]
                    else:
                        # FIXME: not sure if there is a better way
                        layer.is_train = is_train if is_train is not None else self.is_train
                        # FIXME: assume each layer has only one prev layer
                        # z = layer.forward(z)
                        z = layer(z)
                        memory[layer.name] = z
                results.append(z)

            if not isinstance(self._outputs, list):
                return results[0]
            else:
                return results
        else:
            # self._inputs and self._outputs are NOT defined when self is created (eager mode)

            attr_list = [attr for attr in dir(self) if attr[:2] != "__"]
            attr_list.remove("weights")
            for idx, attr in enumerate(attr_list):
                try:
                    if isinstance(getattr(self, attr), Layer):
                        getattr(self, attr).is_train = is_train if is_train is not None else self.is_train
                except Exception:
                    pass

            return self.forward(inputs, **kwargs)

    @property
    def weights(self):
        if self._weights is not None and len(self._weights) > 0:
            # self._weights already extracted, so do nothing
            pass
        elif self._outputs is not None:
            # self._inputs and self._outputs are defined when self is created
            self._weights = list()
            outputs_list = self._outputs if isinstance(self._outputs, list) else [self._outputs]
            for out in outputs_list:
                current = out
                while current is not None:
                    if current.weights is not None:
                        self._weights.extend(current.weights)
                    # FIXME: assume each layer has only one prev layer
                    current = current._input_layer
        else:
            # self._inputs and self._outputs are NOT defined when self is created (eager mode)
            self._weights = list()
            attr_list = [attr for attr in dir(self) if attr[:2] != "__"]
            attr_list.remove("weights")
            for idx, attr in enumerate(attr_list):
                try:
                    if isinstance(getattr(self, attr), Layer):
                        self._weights.extend(getattr(self, attr).weights)
                except Exception:
                    pass

        return self._weights

    def train(self):
        self.is_train = True

    def eval(self):
        self.is_train = False

    def test(self):
        self.eval()

    def infer(self):
        self.eval()

    '''
    def _set_mode_for_layers(self, is_train):
        attr_list = [attr for attr in dir(self) if attr[:2] != "__"]
        attr_list.remove("weights")
        for idx, attr in enumerate(attr_list):
            try:
                if isinstance(getattr(self, attr), Layer):
                    getattr(self, attr).is_train = is_train
            except Exception:
                pass
    '''

    def _find_idx_of_inputs(self, target_input):
        """
        Return the index of the target_input in self._inputs.
        Return -1 if not found.

        :param target_input: the input layer needs to be located
        :return:
        """
        if isinstance(self._inputs, list):
            for idx, input in enumerate(self._inputs):
                if input is target_input:
                    return idx
        return -1

    def __str__(self):
        return "  {} ({}) outputs_shape: {}".format(
            self.__class__.__name__, self.name, [tuple(['batch_size'] + o._outputs_shape[1:]) for o in self.outputs]
        )  #_outputs_shape)#outputs.get_shape().as_list())

    def print_all_layers(self):
        for out in self._outputs:
            stacked_layers = list()
            current = out
            while current is not None:
                print(current.name, current == self._inputs)
                stacked_layers.append(current)
                current = current._input_layer
        pass

    ## raise Exceptions for old version codes
    def count_params(self, **kwargs):
        raise Exception("please change count_params --> count_weights")

    def print_params(self, **kwargs):
        raise Exception("please change print_params --> print_weights")

    @property
    def all_params(self):
        raise Exception("please change all_params --> weights")

    @property
    def all_drop(self):
        raise Exception("all_drop is deprecated")
<|endoftext|>