<commit_before>from typing import Union, Optional, Iterator, Iterable
import random
import time
import traceback

from rlbot.training.status_rendering import training_status_renderer_context, Row
from rlbot.matchconfig.match_config import MatchConfig
from rlbot.setup_manager import SetupManager
from rlbot.utils import rate_limiter
from rlbot.utils.game_state_util import GameState
from rlbot.utils.logging_utils import get_logger, DEFAULT_LOGGER
from rlbot.utils.rendering.rendering_manager import RenderingManager
from rlbot.utils.structures.game_data_struct import GameTickPacket
from rlbot.utils.structures.game_interface import GameInterface

"""
This file contains a minimal API to implement training.
For a more useful API see: https://github.com/RLBot/RLBotTraining/
"""

# Extend Pass and/or Fail to add your own, more detailed metrics.

class Pass:
    """ Indicates that the bot passed the exercise. """

    def __repr__(self):
        return 'PASS'

class Fail:
    """ Indicates that the bot failed the exercise. """

    def __repr__(self):
        return 'FAIL'

class FailDueToExerciseException(Fail):
    """ Indicates that the test code threw an expetion. """

    def __init__(self, exception: Exception, traceback_string: str):
        self.exception = exception
        self.traceback_string = traceback_string

    def __repr__(self):
        return 'FAIL: Exception raised by Exercise:\n' + self.traceback_string

# Note: not using Grade as a abstract base class for Pass/Fail
#       as there should not be Grades which are neither Pass nor Fail.
Grade = Union[Pass, Fail]

class Exercise:
    """
    Statisfy this interface to define your test cases.
    This class provides a seeded random generator to support variation testing.
    The responsibility of detecting timeouts lies with the implementation of
    on_tick().
    """

    def get_name(self) -> str:
        """
        Gets the name to be displayed on screen.
        """
        raise NotImplementedError()

    def get_match_config(self) -> MatchConfig:
        """
        Gets the config with which this exercise should be run.
        """
        raise NotImplementedError()

    def setup(self, rng: random.Random) -> GameState:
        """
        Returns the state in which the game should start in.
        The implementing class is responsible for resetting the state after setup is called,
        such that the exercise can be run multiple times to get the same result.
        :param random: A seeded random number generator. For repeated runs of this
            exercise, this parameter and the bots should be the only things which
            causes variations between runs.
        """
        raise NotImplementedError()

    def on_tick(self, game_tick_packet: GameTickPacket) -> Optional[Grade]:
        """
        This method is called each tick to allow you to make an assessment of the
        performance of the bot(s).
        The order for whether on_tick comes before the bots recieving the packet is undefined.

        If this method returns None, the run of the exercise will continue.
        If this method returns Pass() or Fail() or raises an exceptions, the run of
        the exercise is terminated and any metrics will be returned.
        """
        raise NotImplementedError()

    def render(self, renderer: RenderingManager):
        """
        This method is called each tick to render exercise debug information.
        This method is called after on_tick().
        It is optional to override this method.
        """
        pass

class Result:
    def __init__(self, input_exercise: Exercise, input_seed: int, grade: Grade):
        assert grade
        self.seed = input_seed
        self.exercise = input_exercise
        self.grade = grade

def run_exercises(setup_manager: SetupManager, exercises: Iterable[Exercise], seed: int) -> Iterator[Result]:
    """
    It is recommended to use setup_manager_context() to generate your setup_manager.
    """
    game_interface = setup_manager.game_interface
    names = [exercise.get_name() for exercise in exercises]
    with training_status_renderer_context(names, game_interface.renderer) as ren:
        for i, exercise in enumerate(exercises):
            def update_row(status: str, status_color_func):
                nonlocal i
                nonlocal exercise
                ren.update(i, Row(exercise.get_name(), status, status_color_func))

            update_row('config', ren.renderman.white)
            # Only reload the match if the config has changed.
            new_match_config = exercise.get_match_config()
            if new_match_config != setup_manager.match_config:
                update_row('match', ren.renderman.white)
                _setup_match(new_match_config, setup_manager)
                update_row('bots', ren.renderman.white)
                _wait_until_bots_ready(setup_manager, new_match_config)

            update_row('wait', ren.renderman.white)
            _wait_until_good_ticks(game_interface)

            update_row('setup', ren.renderman.white)
            error_result = _setup_exercise(game_interface, exercise, seed)
            if error_result is not None:
                update_row('setup', ren.renderman.red)
                yield error_result
                continue

            update_row('reload', ren.renderman.white)
            setup_manager.reload_all_agents(quiet=True)

            # Wait for the set_game_state() to propagate before we start running ex.on_tick()
            # TODO: wait until the game looks similar.
            update_row('sleep', ren.renderman.white)
            time.sleep(0.03)

            update_row('>>>>', ren.renderman.white)
            result = _grade_exercise(game_interface, exercise, seed)

            if isinstance(result.grade, Pass):
                update_row('PASS', ren.renderman.green)
            else:
                update_row('FAIL', ren.renderman.red)

            yield result

def _wait_until_bots_ready(setup_manager: SetupManager, match_config: MatchConfig):
    total_ready = 0
    total_ready += setup_manager.try_recieve_agent_metadata()
    logger = get_logger(DEFAULT_LOGGER)
    expected_metadata_calls = sum(1 for player in match_config.player_configs if player.rlbot_controlled)
    while total_ready < expected_metadata_calls:
        logger.debug('Waiting on all bots to post their metadata.')
        time.sleep(0.1)
        total_ready += setup_manager.try_recieve_agent_metadata()

def _wait_until_good_ticks(game_interface: GameInterface, required_new_ticks: int=3):
    """Blocks until we're getting new packets, indicating that the match is ready."""
    rate_limit = rate_limiter.RateLimiter(120)
    last_tick_game_time = None  # What the tick time of the last observed tick was
    game_tick_packet = GameTickPacket()  # We want to do a deep copy for game inputs so people don't mess with em
    seen_times = 0
    while seen_times < required_new_ticks:

        # Read from game data shared memory
        game_interface.update_live_data_packet(game_tick_packet)
        tick_game_time = game_tick_packet.game_info.seconds_elapsed
        if tick_game_time != last_tick_game_time and game_tick_packet.game_info.is_round_active:
            last_tick_game_time = tick_game_time
            seen_times += 1

        rate_limit.acquire()

def _setup_match(match_config: MatchConfig, manager: SetupManager):
    manager.shut_down(quiet=True)  # To be safe.
    manager.load_match_config(match_config)
    manager.launch_quick_chat_manager()
    manager.launch_ball_prediction()
    manager.launch_bot_processes()
    manager.start_match()

def _setup_exercise(game_interface: GameInterface, ex: Exercise, seed: int) -> Optional[Result]:
    """
    Set the game state.
    Only returns a result if there was an error in ex.setup()
    """
    rng = random.Random()
    rng.seed(seed)
    try:
        game_state = ex.setup(rng)
    except Exception as e:
        return Result(ex, seed, FailDueToExerciseException(e, traceback.format_exc()))
    game_interface.set_game_state(game_state)

def _grade_exercise(game_interface: GameInterface, ex: Exercise, seed: int) -> Result:
    grade = None
    rate_limit = rate_limiter.RateLimiter(120)
    last_tick_game_time = None  # What the tick time of the last observed tick was
    game_tick_packet = GameTickPacket()  # We want to do a deep copy for game inputs so people don't mess with em

    # Run until the Exercise finishes.
    while grade is None:

        # Read from game data shared memory
        game_interface.update_live_data_packet(game_tick_packet)

        # Run ex.on_tick() only if the game_info has updated.
        tick_game_time = game_tick_packet.game_info.seconds_elapsed
        if tick_game_time != last_tick_game_time:
            last_tick_game_time = tick_game_time
            try:
                grade = ex.on_tick(game_tick_packet)
                ex.render(game_interface.renderer)
            except Exception as e:
                return Result(ex, seed, FailDueToExerciseException(e, traceback.format_exc()))

        rate_limit.acquire()

    return Result(ex, seed, grade)
<commit_msg>Wait for bot respawn in training (#394)<commit_after>from typing import Union, Optional, Iterator, Iterable
import random
import time
import traceback

from rlbot.training.status_rendering import training_status_renderer_context, Row
from rlbot.matchconfig.match_config import MatchConfig
from rlbot.setup_manager import SetupManager
from rlbot.utils import rate_limiter
from rlbot.utils.game_state_util import GameState
from rlbot.utils.logging_utils import get_logger, DEFAULT_LOGGER
from rlbot.utils.rendering.rendering_manager import RenderingManager
from rlbot.utils.structures.game_data_struct import GameTickPacket
from rlbot.utils.structures.game_interface import GameInterface

"""
This file contains a minimal API to implement training.
For a more useful API see: https://github.com/RLBot/RLBotTraining/
"""

# Extend Pass and/or Fail to add your own, more detailed metrics.

class Pass:
    """ Indicates that the bot passed the exercise. """

    def __repr__(self):
        return 'PASS'

class Fail:
    """ Indicates that the bot failed the exercise. """

    def __repr__(self):
        return 'FAIL'

class FailDueToExerciseException(Fail):
    """ Indicates that the test code threw an expetion. """

    def __init__(self, exception: Exception, traceback_string: str):
        self.exception = exception
        self.traceback_string = traceback_string

    def __repr__(self):
        return 'FAIL: Exception raised by Exercise:\n' + self.traceback_string

# Note: not using Grade as a abstract base class for Pass/Fail
#       as there should not be Grades which are neither Pass nor Fail.
Grade = Union[Pass, Fail]

class Exercise:
    """
    Statisfy this interface to define your test cases.
    This class provides a seeded random generator to support variation testing.
    The responsibility of detecting timeouts lies with the implementation of
    on_tick().
    """

    def get_name(self) -> str:
        """
        Gets the name to be displayed on screen.
        """
        raise NotImplementedError()

    def get_match_config(self) -> MatchConfig:
        """
        Gets the config with which this exercise should be run.
        """
        raise NotImplementedError()

    def setup(self, rng: random.Random) -> GameState:
        """
        Returns the state in which the game should start in.
        The implementing class is responsible for resetting the state after setup is called,
        such that the exercise can be run multiple times to get the same result.
        :param random: A seeded random number generator. For repeated runs of this
            exercise, this parameter and the bots should be the only things which
            causes variations between runs.
        """
        raise NotImplementedError()

    def on_tick(self, game_tick_packet: GameTickPacket) -> Optional[Grade]:
        """
        This method is called each tick to allow you to make an assessment of the
        performance of the bot(s).
        The order for whether on_tick comes before the bots recieving the packet is undefined.

        If this method returns None, the run of the exercise will continue.
        If this method returns Pass() or Fail() or raises an exceptions, the run of
        the exercise is terminated and any metrics will be returned.
        """
        raise NotImplementedError()

    def render(self, renderer: RenderingManager):
        """
        This method is called each tick to render exercise debug information.
        This method is called after on_tick().
        It is optional to override this method.
        """
        pass

class Result:
    def __init__(self, input_exercise: Exercise, input_seed: int, grade: Grade):
        assert grade
        self.seed = input_seed
        self.exercise = input_exercise
        self.grade = grade

def run_exercises(setup_manager: SetupManager, exercises: Iterable[Exercise], seed: int) -> Iterator[Result]:
    """
    It is recommended to use setup_manager_context() to generate your setup_manager.
    """
    game_interface = setup_manager.game_interface
    names = [exercise.get_name() for exercise in exercises]
    with training_status_renderer_context(names, game_interface.renderer) as ren:
        for i, exercise in enumerate(exercises):
            def update_row(status: str, status_color_func):
                nonlocal i
                nonlocal exercise
                ren.update(i, Row(exercise.get_name(), status, status_color_func))

            update_row('config', ren.renderman.white)
            # Only reload the match if the config has changed.
            new_match_config = exercise.get_match_config()
            if new_match_config != setup_manager.match_config:
                update_row('match', ren.renderman.white)
                _setup_match(new_match_config, setup_manager)
                update_row('bots', ren.renderman.white)
                _wait_until_bots_ready(setup_manager, new_match_config)

            update_row('wait', ren.renderman.white)
            _wait_until_good_ticks(game_interface)

            update_row('setup', ren.renderman.white)
            error_result = _setup_exercise(game_interface, exercise, seed)
            if error_result is not None:
                update_row('setup', ren.renderman.red)
                yield error_result
                continue

            update_row('reload', ren.renderman.white)
            setup_manager.reload_all_agents(quiet=True)

            # Wait for the set_game_state() to propagate before we start running ex.on_tick()
            # TODO: wait until the game looks similar.
            update_row('sleep', ren.renderman.white)
            time.sleep(0.03)

            update_row('>>>>', ren.renderman.white)
            result = _grade_exercise(game_interface, exercise, seed)

            if isinstance(result.grade, Pass):
                update_row('PASS', ren.renderman.green)
            else:
                update_row('FAIL', ren.renderman.red)

            yield result

def _wait_until_bots_ready(setup_manager: SetupManager, match_config: MatchConfig):
    total_ready = 0
    total_ready += setup_manager.try_recieve_agent_metadata()
    logger = get_logger(DEFAULT_LOGGER)
    expected_metadata_calls = sum(1 for player in match_config.player_configs if player.rlbot_controlled)
    while total_ready < expected_metadata_calls:
        logger.debug('Waiting on all bots to post their metadata.')
        time.sleep(0.1)
        total_ready += setup_manager.try_recieve_agent_metadata()

def _wait_until_good_ticks(game_interface: GameInterface, required_new_ticks: int=3):
    """Blocks until we're getting new packets, indicating that the match is ready."""
    rate_limit = rate_limiter.RateLimiter(120)
    last_tick_game_time = None  # What the tick time of the last observed tick was
    packet = GameTickPacket()  # We want to do a deep copy for game inputs so people don't mess with em
    seen_times = 0
    while seen_times < required_new_ticks:
        game_interface.update_live_data_packet(packet)
        def is_good_tick():
            if packet.game_info.seconds_elapsed == last_tick_game_time: return False
            if not packet.game_info.is_round_active: return False
            if any(car.is_demolished for car in packet.game_cars): return False
            return True
        if is_good_tick():
            seen_times += 1
        last_tick_game_time = packet.game_info.seconds_elapsed

        rate_limit.acquire()

def _setup_match(match_config: MatchConfig, manager: SetupManager):
    manager.shut_down(quiet=True)  # To be safe.
    manager.load_match_config(match_config)
    manager.launch_quick_chat_manager()
    manager.launch_ball_prediction()
    manager.launch_bot_processes()
    manager.start_match()

def _setup_exercise(game_interface: GameInterface, ex: Exercise, seed: int) -> Optional[Result]:
    """
    Set the game state.
    Only returns a result if there was an error in ex.setup()
    """
    rng = random.Random()
    rng.seed(seed)
    try:
        game_state = ex.setup(rng)
    except Exception as e:
        return Result(ex, seed, FailDueToExerciseException(e, traceback.format_exc()))
    game_interface.set_game_state(game_state)

def _grade_exercise(game_interface: GameInterface, ex: Exercise, seed: int) -> Result:
    grade = None
    rate_limit = rate_limiter.RateLimiter(120)
    last_tick_game_time = None  # What the tick time of the last observed tick was
    game_tick_packet = GameTickPacket()  # We want to do a deep copy for game inputs so people don't mess with em

    # Run until the Exercise finishes.
    while grade is None:

        # Read from game data shared memory
        game_interface.update_live_data_packet(game_tick_packet)

        # Run ex.on_tick() only if the game_info has updated.
        tick_game_time = game_tick_packet.game_info.seconds_elapsed
        if tick_game_time != last_tick_game_time:
            last_tick_game_time = tick_game_time
            try:
                grade = ex.on_tick(game_tick_packet)
                ex.render(game_interface.renderer)
            except Exception as e:
                return Result(ex, seed, FailDueToExerciseException(e, traceback.format_exc()))

        rate_limit.acquire()

    return Result(ex, seed, grade)
<|endoftext|>


<commit_before>import luigi
import sciluigi as sl
import os

TESTFILE_PATH = '/tmp/test.out'

class TestTask(sl.Task):

    def out_data(self):
        return sl.TargetInfo(self, TESTFILE_PATH)

    def run(self):
        with self.out_data().open('w') as outfile:
            outfile.write('File written by luigi\n')

class TestRunTask():

    def setup(self):
        self.t = sl.new_task(TestTask)

    def teardown(self):
        self.t = None
        os.remove(TESTFILE_PATH)

    def test_run(self):
        # Run a task with a luigi worker
        w = luigi.worker.Worker()
        w.add(self.t)
        w.run()
        w.stop()

        assert os.path.isfile(TESTFILE_PATH)
<commit_msg>Make sure test works with new API<commit_after>import luigi
import sciluigi as sl
import os

TESTFILE_PATH = '/tmp/test.out'

class TestTask(sl.Task):

    def out_data(self):
        return sl.TargetInfo(self, TESTFILE_PATH)

    def run(self):
        with self.out_data().open('w') as outfile:
            outfile.write('File written by luigi\n')

class TestRunTask():

    def setup(self):
        self.t = sl.new_task(self, TestTask, 'testtask')

    def teardown(self):
        self.t = None
        os.remove(TESTFILE_PATH)

    def test_run(self):
        # Run a task with a luigi worker
        w = luigi.worker.Worker()
        w.add(self.t)
        w.run()
        w.stop()

        assert os.path.isfile(TESTFILE_PATH)
<|endoftext|>


<commit_before>import flask
import destinate

import os

SECRET = os.environ.get('SERVER_SECRET') or 'default_secret_key'
SESSION_TOKEN = 'user-session-token'

app = flask.Flask(__name__)
app.secret_key = SECRET
app.debug=True

def auth(func):
    def auth_and_run(*args, **kwargs):
        token = flask.session.get(SESSION_TOKEN)
        if token and authenticate(token):
            return func(*args, **kwargs)
        return flask.abort(403)
    return auth_and_run

def authenticate(token):
    return destinate.profile.is_valid_token(token)

@app.route('/logins/',methods=['POST'])
def login():
    token = flask.request.form['token']
    try:
        user = destinate.profile.find_and_analyze_user(token)
    except ValueError:
        return flask.abort(400)
    response = flask.make_response(flask.jsonify(user=user))
    flask.session[SESSION_TOKEN] = token
    return response

@app.route('/suggestions/')
@auth
def suggest():
    user = destinate.profile.get_user(flask.session[SESSION_TOKEN])
    months=flask.request.values.getlist('month')
    regions=flask.request.values.getlist('region')

    guide = user['summary']['events'] + '\n' + u', '.join(user['summary']['topics'])
    cities = destinate.suggest.from_guide(
        guide,
        months=months,
        ignore_name=user.get('city'))
    return flask.jsonify(suggestions=cities)
    <commit_msg>enable jsonp requests<commit_after>import flask
import destinate

import os

SECRET = os.environ.get('SERVER_SECRET') or 'default_secret_key'
SESSION_TOKEN = 'user-session-token'

app = flask.Flask(__name__)
app.secret_key = SECRET
app.debug=True

def auth(func):
    def auth_and_run(*args, **kwargs):
        token = flask.session.get(SESSION_TOKEN)
        if token and authenticate(token):
            return func(*args, **kwargs)
        return flask.abort(403)
    return auth_and_run

def authenticate(token):
    return destinate.profile.is_valid_token(token)

def enable_jsonp_response(*args, **kwargs):
    callback = flask.request.args.get('callback')
    json_str = flask.json.dumps(*args, **kwargs)
    if callback:
        json_str = '{}({});'.format(callback, json_str)
    response = flask.make_response(json_str)
    response.headers['Content-Type'] = 'application/json'
    return response

@app.route('/logins/',methods=['POST'])
def login():
    token = flask.request.form['token']
    try:
        user = destinate.profile.find_and_analyze_user(token)
    except ValueError:
        return flask.abort(400)
    response = enable_jsonp_response(user=user)
    flask.session[SESSION_TOKEN] = token
    return response

@app.route('/suggestions/')
@auth
def suggest():
    user = destinate.profile.get_user(flask.session[SESSION_TOKEN])
    months=flask.request.values.getlist('month')
    regions=flask.request.values.getlist('region')

    guide = user['summary']['events'] + '\n' + u', '.join(user['summary']['topics'])
    cities = destinate.suggest.from_guide(
        guide,
        months=months,
        ignore_name=user.get('city'))
    return enable_jsonp_response(suggestions=cities)
    <|endoftext|>


<commit_before>#!/usr/bin/env python
# Web frontend for CVE -> RHSA report generator.
#
# Requires CherryPy, Mako, Python <= 2.7

import rhsa
import cherrypy
import os
from mako.template import Template
from mako.lookup import TemplateLookup

tlu = TemplateLookup(directories=['templates'])

curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))

cherrypy.config.update({
    'server.socket_port': 8090,
    'server.socket_host': '0.0.0.0',
    'tools.staticdir.root': curdir
})

class RHSAGenWeb:
    @cherrypy.expose
    def index(self):
        return tlu.get_template("index.html").render(rhelver=rhsa.RHEL_VERSION)

    @cherrypy.expose
    def repgen(self, cves=None):
        if cves == None or cves == "":
            return "You didn't give me a list of CVEs :("

        rhsalist = []

        cves = cves.split()

        for cve in cves:
            rhsalist.append(rhsa.get_cve_info(cve))

        return tlu.get_template("repgen.html").render(rhsalist=rhsalist)

if __name__ == "__main__":
    cherrypy.quickstart(RHSAGenWeb(), "/", "web.conf")
<commit_msg>Removed redundant configuration options.<commit_after>#!/usr/bin/env python
# Web frontend for CVE -> RHSA report generator.
#
# Requires CherryPy, Mako, Python <= 2.7

import rhsa
import cherrypy
import os
from mako.template import Template
from mako.lookup import TemplateLookup

tlu = TemplateLookup(directories=['templates'])

curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))

cherrypy.config.update({
    'tools.staticdir.root': curdir
})

class RHSAGenWeb:
    @cherrypy.expose
    def index(self):
        return tlu.get_template("index.html").render(rhelver=rhsa.RHEL_VERSION)

    @cherrypy.expose
    def repgen(self, cves=None):
        if cves == None or cves == "":
            return "You didn't give me a list of CVEs :("

        rhsalist = []

        cves = cves.split()

        for cve in cves:
            rhsalist.append(rhsa.get_cve_info(cve))

        return tlu.get_template("repgen.html").render(rhsalist=rhsalist)

if __name__ == "__main__":
    cherrypy.quickstart(RHSAGenWeb(), "/", "web.conf")
<|endoftext|>


<commit_before>from distutils.core import setup

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

setup(name='DataHandling',
      version='1.5.0',
      description='Python package with functions for data analysis',
      author='Ashley Setter',
      author_email='A.Setter@soton.ac.uk',
      url=None,
      package_dir={'DataHandling': 'DataHandling',
                   'DataHandling.SimData': 'DataHandling/SimData',
      },
      packages=['DataHandling', 'DataHandling.LeCroy', 'Datahandling.SimData'],
      install_requires=requirements,
)
<commit_msg>trying to fix travis pip installation<commit_after>from distutils.core import setup

with open('requirements.txt') as f:
    requirements = f.read().splitlines()

setup(name='DataHandling',
      version='1.5.0',
      description='Python package with functions for data analysis',
      author='Ashley Setter',
      author_email='A.Setter@soton.ac.uk',
      url=None,
      package_dir={'DataHandling': 'DataHandling',
                   'DataHandling.SimData': 'DataHandling/DataHandling/SimData',
      },
      packages=['DataHandling', 'DataHandling.LeCroy', 'Datahandling.SimData'],
      install_requires=requirements,
)
<|endoftext|>


<commit_before>import gdb

from .cmd import SimpleCommand
from .struct import GdbStructMeta
from struct import *

def gmon_write(path):
    class GmonParam(metaclass=GdbStructMeta):
        __ctype__ = 'struct gmonparam'

    gparam = GmonParam(gdb.parse_and_eval('_gmonparam'))
    infer = gdb.inferiors()[0]

    with open(path, "wb") as of:
        # Write headers
        gmonhdr_size = int(gdb.parse_and_eval('sizeof(_gmonhdr)'))
        gmonhdr_p = gdb.parse_and_eval('&_gmonhdr')
        of.write(infer.read_memory(gmonhdr_p, gmonhdr_size))

        # Write tick buffer
        of.write(infer.read_memory(gparam.kcount, gparam.kcountsize))

        # Write arc info
        memory = infer.read_memory(gparam.froms, gparam.fromssize)
        froms_array = unpack('H' * int(gparam.fromssize/calcsize('H')), memory)
        memory = infer.read_memory(gparam.tos, gparam.tossize)

        # The last H stands for padding in the tos strusture
        tos_rep = 'IiHH'
        tos_rep_len = len(tos_rep)
        size = calcsize(tos_rep)
        tos_array = unpack(tos_rep * int(gparam.tossize/size), memory)

        fromindex = 0
        froms_el_size = int(gdb.parse_and_eval('sizeof(*_gmonparam.froms)'))
        for from_val in froms_array:
            # Nothing has been called from this function
            if from_val == 0:
                continue
            # Getting the calling function addres from encoded value
            offset = fromindex * froms_el_size * gparam.hashfraction
            frompc = gparam.lowpc + offset
            toindex = from_val

            # Traversing the tos list for the calling function
            # It stores data about called functions
            while toindex != 0:
                selfpc = tos_array[toindex * tos_rep_len]
                count = tos_array[toindex * tos_rep_len + 1]
                toindex = tos_array[toindex * tos_rep_len + 2]
                of.write(pack('IIi', frompc, selfpc, count))
            fromindex += 1

class Kgmon(SimpleCommand):
    """Dump the gprof data to file (by default 'gmon.out')"""

    def __init__(self):
        super().__init__('kgmon')

    def __call__(self, args):
        args = args.strip()
        state = gdb.parse_and_eval('_gmonparam.state')
        if state == gdb.parse_and_eval('GMON_PROF_NOT_INIT'):
            print("Kgprof not initialized yet")
        elif state == gdb.parse_and_eval('GMON_PROF_BUSY'):
            # To ensure consistent data
            print("The mcount function is running - wait for it to finish")
        else:
            if state == gdb.parse_and_eval('GMON_PROF_ERROR'):
                print("The tostruct array was too small for the whole process")
            gmon_write(args or 'gmon.out')
<commit_msg>kgmon: fix call graph construction for leaf functions (#1155)<commit_after>import gdb

from .cmd import SimpleCommand
from .struct import GdbStructMeta
from struct import *

def gmon_write(path):
    class GmonParam(metaclass=GdbStructMeta):
        __ctype__ = 'struct gmonparam'

    gparam = GmonParam(gdb.parse_and_eval('_gmonparam'))
    infer = gdb.inferiors()[0]

    with open(path, "wb") as of:
        # Write headers
        gmonhdr_size = int(gdb.parse_and_eval('sizeof(_gmonhdr)'))
        gmonhdr_p = gdb.parse_and_eval('&_gmonhdr')
        of.write(infer.read_memory(gmonhdr_p, gmonhdr_size))

        # Write tick buffer
        of.write(infer.read_memory(gparam.kcount, gparam.kcountsize))

        # Write arc info
        memory = infer.read_memory(gparam.froms, gparam.fromssize)
        froms_array = unpack('H' * int(gparam.fromssize/calcsize('H')), memory)
        memory = infer.read_memory(gparam.tos, gparam.tossize)

        # The last H stands for padding in the tos strusture
        tos_rep = 'IiHH'
        tos_rep_len = len(tos_rep)
        size = calcsize(tos_rep)
        tos_array = unpack(tos_rep * int(gparam.tossize/size), memory)

        fromindex = -1
        froms_el_size = int(gdb.parse_and_eval('sizeof(*_gmonparam.froms)'))
        for from_val in froms_array:
            fromindex += 1
            # Nothing has been called from this function
            if from_val == 0:
                continue
            # Getting the calling function addres from encoded value
            offset = fromindex * froms_el_size * gparam.hashfraction
            frompc = gparam.lowpc + offset
            toindex = from_val

            # Traversing the tos list for the calling function
            # It stores data about called functions
            while toindex != 0:
                selfpc = tos_array[toindex * tos_rep_len]
                count = tos_array[toindex * tos_rep_len + 1]
                toindex = tos_array[toindex * tos_rep_len + 2]
                of.write(pack('IIi', frompc, selfpc, count))

class Kgmon(SimpleCommand):
    """Dump the gprof data to file (by default 'gmon.out')"""

    def __init__(self):
        super().__init__('kgmon')

    def __call__(self, args):
        args = args.strip()
        state = gdb.parse_and_eval('_gmonparam.state')
        if state == gdb.parse_and_eval('GMON_PROF_NOT_INIT'):
            print("Kgprof not initialized yet")
        elif state == gdb.parse_and_eval('GMON_PROF_BUSY'):
            # To ensure consistent data
            print("The mcount function is running - wait for it to finish")
        else:
            if state == gdb.parse_and_eval('GMON_PROF_ERROR'):
                print("The tostruct array was too small for the whole process")
            gmon_write(args or 'gmon.out')
<|endoftext|>


<commit_before>'''
Created on Apr 25, 2015

@author: Sherry
'''

from json import dumps

from twitterDataAcquisition import TwitterDataAcquisition
from twitterScreenScrape import TwitterScreenScrape

def main():
    appsList = ["Messenger", "Criminal Case", "Facebook", "Pandora Radio", "Instagram",
            "Snapchat", "Dubsmash","Super-Bright LED Flashlight", "Spotify Music",
            "Clean Master (Speed Booster)"]

    #twitterData = TwitterDataAcquisition()

    #jsonFile = ("twitter_data.json", "w")
    #jsonFile.write(str(twitterData))
    #jsonFile.close()

    #print dumps(twitterData)

    TwitterScreenScrape(appsList)
    #TwitterScreenScrape(["Messenger"])

if __name__ == '__main__':
    main()<commit_msg>Delete testTwitterAPI.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

""" Module to test exercise1.py """

__author__ = 'Susan Sim'
__email__ = "ses@drsusansim.org"

__copyright__ = "2014 Susan Sim"
__license__ = "MIT License"

__status__ = "Prototype"

# imports one per line
import pytest
from exercise1 import grade_to_gpa

def test_letter_grade():
    """
    Letter grade inputs
    """
    assert grade_to_gpa("A+") == 4.0
    assert grade_to_gpa("A") == 4.0
    assert grade_to_gpa("A-") == 4.0
    assert grade_to_gpa("B+") == 4.0
    assert grade_to_gpa("B") == 4.0
    assert grade_to_gpa("B-") == 4.0
    assert grade_to_gpa("FZ") == 4.0

    with pytest.raises(ValueError):
        grade_to_gpa("q")
    # add more tests for invalid values

def test_percentage_grade():
    """
    Numeric grade inputs
    """
    assert grade_to_gpa(100) == 4.0
    assert grade_to_gpa(95) == 4.0
    assert grade_to_gpa(90) == 4.0

    assert grade_to_gpa(89) == 4.0
    assert grade_to_gpa(87) == 4.0
    assert grade_to_gpa(85) == 4.0

    assert grade_to_gpa(84) == 3.7
    assert grade_to_gpa(82) == 3.7
    assert grade_to_gpa(80) == 3.7

    assert grade_to_gpa(79) == 3.3
    assert grade_to_gpa(78) == 3.3
    assert grade_to_gpa(77) == 3.3

    assert grade_to_gpa(76) == 3.0
    assert grade_to_gpa(74) == 3.0
    assert grade_to_gpa(73) == 3.0

    assert grade_to_gpa(72) == 2.7
    assert grade_to_gpa(71) == 2.7
    assert grade_to_gpa(70) == 2.7

    assert grade_to_gpa(69) == 0.0
    assert grade_to_gpa(37) == 0.0
    assert grade_to_gpa(0) == 0.0

    with pytest.raises(ValueError):
        grade_to_gpa(101)
        grade_to_gpa(-1)

def test_float_input():
    """
    Float inputs
    """
    with pytest.raises(TypeError):
        grade_to_gpa(82.5)

# add functions for any other tests
<commit_msg>Corrected test cases for letter grades in Exercise 1<commit_after>#!/usr/bin/env python3

""" Module to test exercise1.py """

__author__ = 'Susan Sim'
__email__ = "ses@drsusansim.org"

__copyright__ = "2014 Susan Sim"
__license__ = "MIT License"

__status__ = "Prototype"

# imports one per line
import pytest
from exercise1 import grade_to_gpa

def test_letter_grade():
    """
    Letter grade inputs
    """
    assert grade_to_gpa("A+") == 4.0
    assert grade_to_gpa("A") == 4.0
    assert grade_to_gpa("A-") == 3.7
    assert grade_to_gpa("B+") == 3.3
    assert grade_to_gpa("B") == 3.0
    assert grade_to_gpa("B-") == 2.7
    assert grade_to_gpa("FZ") == 0.0

    with pytest.raises(ValueError):
        grade_to_gpa("q")
    # add more tests for invalid values

def test_percentage_grade():
    """
    Numeric grade inputs
    """
    assert grade_to_gpa(100) == 4.0
    assert grade_to_gpa(95) == 4.0
    assert grade_to_gpa(90) == 4.0

    assert grade_to_gpa(89) == 4.0
    assert grade_to_gpa(87) == 4.0
    assert grade_to_gpa(85) == 4.0

    assert grade_to_gpa(84) == 3.7
    assert grade_to_gpa(82) == 3.7
    assert grade_to_gpa(80) == 3.7

    assert grade_to_gpa(79) == 3.3
    assert grade_to_gpa(78) == 3.3
    assert grade_to_gpa(77) == 3.3

    assert grade_to_gpa(76) == 3.0
    assert grade_to_gpa(74) == 3.0
    assert grade_to_gpa(73) == 3.0

    assert grade_to_gpa(72) == 2.7
    assert grade_to_gpa(71) == 2.7
    assert grade_to_gpa(70) == 2.7

    assert grade_to_gpa(69) == 0.0
    assert grade_to_gpa(37) == 0.0
    assert grade_to_gpa(0) == 0.0

    with pytest.raises(ValueError):
        grade_to_gpa(101)
        grade_to_gpa(-1)

def test_float_input():
    """
    Float inputs
    """
    with pytest.raises(TypeError):
        grade_to_gpa(82.5)

# add functions for any other tests
<|endoftext|>


<commit_before>
from discord.ext import commands

import difflib

import asyncpg

from utilities import checks

def setup(bot):
	bot.add_cog(Blobs(bot))

class Blobs(commands.Cog):

	def __init__(self, bot):
		self.bot = bot
		self.bot.loop.create_task(self.initialize_database())

	async def initialize_database(self):
		await self.bot.connect_to_database()
		await self.bot.db.execute("CREATE SCHEMA IF NOT EXISTS blobs")
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.blobs (
				blob	TEXT PRIMARY KEY,
				image	TEXT
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.aliases (
				alias	TEXT PRIMARY KEY,
				blob	TEXT REFERENCES blobs.blobs(blob) ON DELETE CASCADE
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.stats (
				blob			TEXT REFERENCES blobs.blobs(blob) ON DELETE CASCADE,
				user_id			BIGINT,
				count			BIGINT,
				PRIMARY KEY		(blob, user_id)
			)
			"""
		)

	@commands.group(aliases = ["blob"], invoke_without_command = True)
	@checks.not_forbidden()
	async def blobs(self, ctx, *, blob : str):
		'''Blob/Google Emoji'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs UNION SELECT alias FROM blobs.aliases")
		blob_names = [record["blob"] for record in records]
		close_match = difflib.get_close_matches(blob, blob_names, n = 1)
		if not close_match:
			return await ctx.embed_reply(":no_entry: Blob not found")
		blob = close_match[0]
		image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", blob)
		if not image_url:
			blob = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", blob)
			image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", blob)
		await ctx.embed_reply(title = blob, image_url = image_url)
		await ctx.bot.db.execute(
			"""
			INSERT INTO blobs.stats (blob, user_id, count)
			VALUES ($1, $2, 1)
			ON CONFLICT (blob, user_id) DO
			UPDATE SET count = stats.count + 1
			""",
			blob, ctx.author.id
		)

	@blobs.command(aliases = ["edit"])
	@commands.is_owner()
	async def add(self, ctx, name : str, image_url : str, *aliases : str):
		'''Add or edit a blob'''
		await ctx.bot.db.execute(
			"""
			INSERT INTO blobs.blobs (blob, image)
			VALUES ($1, $2)
			ON CONFLICT (blob) DO
			UPDATE SET image = $2
			""",
			name, image_url
		)
		await ctx.bot.db.execute("DELETE FROM blobs.aliases WHERE blob = $1", name)
		for alias in aliases:
			inserted = await ctx.bot.db.fetchrow(
				"""
				INSERT INTO blobs.aliases (alias, blob)
				VALUES ($1, $2)
				ON CONFLICT DO NOTHING
				RETURNING *
				""",
				alias, name
			)
			if not inserted:
				await ctx.embed_reply(f"Failed to add already existing alias: {alias}")
		await ctx.embed_reply("Blob added/edited")

	@blobs.command(aliases = ["details"])
	@commands.is_owner()
	async def info(self, ctx, name : str):
		'''Information about a blob'''
		image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", name)
		if not image_url:
			name = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", name)
			if not name:
				return await ctx.embed_reply(f":no_entry: Blob not found")
			image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", name)
		records = await ctx.bot.db.fetch("SELECT alias FROM blobs.aliases WHERE blob = $1", name)
		aliases = [record["alias"] for record in records]
		await ctx.embed_reply(image_url, title = name, fields = (("Aliases", ", ".join(aliases) or "None"),))

	@blobs.command()
	@checks.not_forbidden()
	async def list(self, ctx):
		'''List blobs'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs")
		blob_names = [record["blob"] for record in records]
		await ctx.embed_reply(", ".join(sorted(blob_names)))

	@blobs.command(aliases = ["delete"])
	@commands.is_owner()
	async def remove(self, ctx, name : str):
		'''Remove a blob'''
		await ctx.bot.db.execute("DELETE FROM blobs.blobs WHERE blob = $1", name)
		await ctx.embed_reply("Blob removed")

	@blobs.command()
	@checks.not_forbidden()
	async def stats(self, ctx, *, blob : str):
		'''Blob emoji stats'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs UNION SELECT alias FROM blobs.aliases")
		blob_names = [record["blob"] for record in records]
		close_match = difflib.get_close_matches(blob, blob_names, n = 1)
		if not close_match:
			return await ctx.embed_reply(":no_entry: Blob not found")
		blob = close_match[0]
		records = await ctx.bot.db.fetch("SELECT user_id, count FROM blobs.stats WHERE blob = $1", blob)
		if not records:
			blob = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", blob)
			records = await ctx.bot.db.fetch("SELECT user_id, count FROM blobs.stats WHERE blob = $1", blob)
		personal = 0
		total = 0
		for record in records:
			if record["user_id"] == ctx.author.id:
				personal = record["count"]
			total += record["count"]
		await ctx.embed_reply(f"Personal: {personal}\nTotal: {total}")

	@blobs.command()
	@checks.not_forbidden()
	async def top(self, ctx):
		'''Top blob emoji'''
		records = await ctx.bot.db.fetch(
			"""
			SELECT blob, count
			FROM blobs.stats
			WHERE user_id = $1
			ORDER BY count
			DESC LIMIT 5
			""",
			ctx.author.id
		)
		personal = [f"{count}. {record['blob']} ({record['count']})" for count, record in enumerate(records, start = 1)]
		records = await ctx.bot.db.fetch(
			"""
			SELECT blob, SUM(count) as count
			FROM blobs.stats
			GROUP BY blob
			ORDER BY SUM(count)
			DESC LIMIT 5
			"""
		)
		total = [f"{count}. {record['blob']} ({record['count']})" for count, record in enumerate(records, start = 1)]
		await ctx.embed_reply(fields = (("Personal", '\n'.join(personal)), ("Total", '\n'.join(total))))

<commit_msg>[Discord] Remove now unnecessary asyncpg import from blobs cog<commit_after>
from discord.ext import commands

import difflib

from utilities import checks

def setup(bot):
	bot.add_cog(Blobs(bot))

class Blobs(commands.Cog):

	def __init__(self, bot):
		self.bot = bot
		self.bot.loop.create_task(self.initialize_database())

	async def initialize_database(self):
		await self.bot.connect_to_database()
		await self.bot.db.execute("CREATE SCHEMA IF NOT EXISTS blobs")
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.blobs (
				blob	TEXT PRIMARY KEY,
				image	TEXT
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.aliases (
				alias	TEXT PRIMARY KEY,
				blob	TEXT REFERENCES blobs.blobs(blob) ON DELETE CASCADE
			)
			"""
		)
		await self.bot.db.execute(
			"""
			CREATE TABLE IF NOT EXISTS blobs.stats (
				blob			TEXT REFERENCES blobs.blobs(blob) ON DELETE CASCADE,
				user_id			BIGINT,
				count			BIGINT,
				PRIMARY KEY		(blob, user_id)
			)
			"""
		)

	@commands.group(aliases = ["blob"], invoke_without_command = True)
	@checks.not_forbidden()
	async def blobs(self, ctx, *, blob : str):
		'''Blob/Google Emoji'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs UNION SELECT alias FROM blobs.aliases")
		blob_names = [record["blob"] for record in records]
		close_match = difflib.get_close_matches(blob, blob_names, n = 1)
		if not close_match:
			return await ctx.embed_reply(":no_entry: Blob not found")
		blob = close_match[0]
		image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", blob)
		if not image_url:
			blob = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", blob)
			image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", blob)
		await ctx.embed_reply(title = blob, image_url = image_url)
		await ctx.bot.db.execute(
			"""
			INSERT INTO blobs.stats (blob, user_id, count)
			VALUES ($1, $2, 1)
			ON CONFLICT (blob, user_id) DO
			UPDATE SET count = stats.count + 1
			""",
			blob, ctx.author.id
		)

	@blobs.command(aliases = ["edit"])
	@commands.is_owner()
	async def add(self, ctx, name : str, image_url : str, *aliases : str):
		'''Add or edit a blob'''
		await ctx.bot.db.execute(
			"""
			INSERT INTO blobs.blobs (blob, image)
			VALUES ($1, $2)
			ON CONFLICT (blob) DO
			UPDATE SET image = $2
			""",
			name, image_url
		)
		await ctx.bot.db.execute("DELETE FROM blobs.aliases WHERE blob = $1", name)
		for alias in aliases:
			inserted = await ctx.bot.db.fetchrow(
				"""
				INSERT INTO blobs.aliases (alias, blob)
				VALUES ($1, $2)
				ON CONFLICT DO NOTHING
				RETURNING *
				""",
				alias, name
			)
			if not inserted:
				await ctx.embed_reply(f"Failed to add already existing alias: {alias}")
		await ctx.embed_reply("Blob added/edited")

	@blobs.command(aliases = ["details"])
	@commands.is_owner()
	async def info(self, ctx, name : str):
		'''Information about a blob'''
		image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", name)
		if not image_url:
			name = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", name)
			if not name:
				return await ctx.embed_reply(f":no_entry: Blob not found")
			image_url = await ctx.bot.db.fetchval("SELECT image FROM blobs.blobs WHERE blob = $1", name)
		records = await ctx.bot.db.fetch("SELECT alias FROM blobs.aliases WHERE blob = $1", name)
		aliases = [record["alias"] for record in records]
		await ctx.embed_reply(image_url, title = name, fields = (("Aliases", ", ".join(aliases) or "None"),))

	@blobs.command()
	@checks.not_forbidden()
	async def list(self, ctx):
		'''List blobs'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs")
		blob_names = [record["blob"] for record in records]
		await ctx.embed_reply(", ".join(sorted(blob_names)))

	@blobs.command(aliases = ["delete"])
	@commands.is_owner()
	async def remove(self, ctx, name : str):
		'''Remove a blob'''
		await ctx.bot.db.execute("DELETE FROM blobs.blobs WHERE blob = $1", name)
		await ctx.embed_reply("Blob removed")

	@blobs.command()
	@checks.not_forbidden()
	async def stats(self, ctx, *, blob : str):
		'''Blob emoji stats'''
		records = await ctx.bot.db.fetch("SELECT blob FROM blobs.blobs UNION SELECT alias FROM blobs.aliases")
		blob_names = [record["blob"] for record in records]
		close_match = difflib.get_close_matches(blob, blob_names, n = 1)
		if not close_match:
			return await ctx.embed_reply(":no_entry: Blob not found")
		blob = close_match[0]
		records = await ctx.bot.db.fetch("SELECT user_id, count FROM blobs.stats WHERE blob = $1", blob)
		if not records:
			blob = await ctx.bot.db.fetchval("SELECT blob FROM blobs.aliases WHERE alias = $1", blob)
			records = await ctx.bot.db.fetch("SELECT user_id, count FROM blobs.stats WHERE blob = $1", blob)
		personal = 0
		total = 0
		for record in records:
			if record["user_id"] == ctx.author.id:
				personal = record["count"]
			total += record["count"]
		await ctx.embed_reply(f"Personal: {personal}\nTotal: {total}")

	@blobs.command()
	@checks.not_forbidden()
	async def top(self, ctx):
		'''Top blob emoji'''
		records = await ctx.bot.db.fetch(
			"""
			SELECT blob, count
			FROM blobs.stats
			WHERE user_id = $1
			ORDER BY count
			DESC LIMIT 5
			""",
			ctx.author.id
		)
		personal = [f"{count}. {record['blob']} ({record['count']})" for count, record in enumerate(records, start = 1)]
		records = await ctx.bot.db.fetch(
			"""
			SELECT blob, SUM(count) as count
			FROM blobs.stats
			GROUP BY blob
			ORDER BY SUM(count)
			DESC LIMIT 5
			"""
		)
		total = [f"{count}. {record['blob']} ({record['count']})" for count, record in enumerate(records, start = 1)]
		await ctx.embed_reply(fields = (("Personal", '\n'.join(personal)), ("Total", '\n'.join(total))))

<|endoftext|>


<commit_before>import warnings
import django
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import connections, DEFAULT_DB_ALIAS
from django.test import TestCase
from distutils.version import LooseVersion

if LooseVersion(django.get_version()) >= LooseVersion('1.6'):
    from django.test.utils import CaptureQueriesContext
    CAPTURE = True

    class _AssertNumQueriesLessThanContext(CaptureQueriesContext):
        def __init__(self, test_case, num, connection):
            self.test_case = test_case
            self.num = num
            super(_AssertNumQueriesLessThanContext, self).__init__(connection)

        def __exit__(self, exc_type, exc_value, traceback):
            super(_AssertNumQueriesLessThanContext, self).__exit__(exc_type, exc_value, traceback)
            if exc_type is not None:
                return
            executed = len(self)
            self.test_case.assertTrue(
                executed <= self.num, "%d queries executed, expected less than %d" % (
                    executed, self.num
                )
            )

else:
    CAPTURE = False

class login(object):
    def __init__(self, testcase, **credentials):
        self.testcase = testcase
        success = testcase.client.login(**credentials)
        self.testcase.assertTrue(
            success,
            "login failed with credentials=%r" % (credentials)
        )

    def __enter__(self):
        pass

    def __exit__(self, *args):
        self.testcase.client.logout()

class TestCase(TestCase):
    """
    Django TestCase with helpful additional features
    """
    user_factory = None

    def tearDown(self):
        self.client.logout()

    def get(self, url_name, *args, **kwargs):
        """ GET url by name using reverse() """
        return self.client.get(reverse(url_name, args=args, kwargs=kwargs))

    def post(self, url_name, *args, **kwargs):
        """ POST to url by name using reverse() """
        data = kwargs.pop("data", None)
        return self.client.post(reverse(url_name, args=args, kwargs=kwargs), data)

    def response_200(self, response):
        """ Given response has status_code 200 """
        self.assertEqual(response.status_code, 200)

    def response_201(self, response):
        """ Given response has status_code 201 """
        self.assertEqual(response.status_code, 201)

    def response_302(self, response):
        """ Given response has status_code 302 """
        self.assertEqual(response.status_code, 302)

    def response_403(self, response):
        """ Given response has status_code 403 """
        self.assertEqual(response.status_code, 403)

    def response_404(self, response):
        """ Given response has status_code 404 """
        self.assertEqual(response.status_code, 404)

    def get_check_200(self, url, *args, **kwargs):
        """ Test that we can GET a page and it returns a 200 """
        response = self.get(url, *args, **kwargs)
        self.response_200(response)
        return response

    def assertLoginRequired(self, url, *args, **kwargs):
        """ Ensure login is required to GET this URL """
        res = self.get(url, *args, **kwargs)
        reversed_url = reverse(url, args=args, kwargs=kwargs)
        expected_url = "{0}?next={1}".format(settings.LOGIN_URL, reversed_url)
        self.assertRedirects(res, expected_url)

    def login(self, **credentials):
        """ Login a user """
        return login(self, **credentials)

    def reverse(self, name, *args, **kwargs):
        """ Reverse a url, convience to avoid having to import reverse in tests """
        return reverse(name, args=args, kwargs=kwargs)

    def make_user(self, username):
        """
        Build a user with <username> and password of 'password' for testing
        purposes.  Exposes the user as self.user_<username>
        """
        if self.user_factory:
            test_user = self.user_factory(username=username)
            test_user.set_password('password')
            test_user.save()
            return test_user
        else:
            if LooseVersion(django.get_version()) >= LooseVersion('1.6'):
                from django.contrib.auth import get_user_model
                User = get_user_model()
            else:
                from django.contrib.auth.models import User

            test_user = User.objects.create_user(
                username,
                '{0}@example.com'.format(username),
                'password',
            )
            return test_user

    def assertNumQueriesLessThan(self, num, func=None, *args, **kwargs):
        if CAPTURE:
            using = kwargs.pop("using", DEFAULT_DB_ALIAS)
            conn = connections[using]

            context = _AssertNumQueriesLessThanContext(self, num, conn)
            if func is None:
                return context

            with context:
                func(*args, **kwargs)
        else:
            warnings.warn("assertNumQueriesLessThan being skipped, does not work prior to Django 1.6")
            func(*args, **kwargs)

    def assertGoodView(self, url_name, *args, **kwargs):
        """
        Quick-n-dirty testing of a given url name.
        Ensures URL returns a 200 status and that generates less than 100
        database queries.
        """
        query_count = kwargs.pop('test_query_count', 100)

        with self.assertNumQueriesLessThan(query_count):
            response = self.get(url_name, *args, **kwargs)

        self.response_200(response)

<commit_msg>Fake context for versions of Django prior to 1.6<commit_after>import warnings
import django
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import connections, DEFAULT_DB_ALIAS
from django.test import TestCase
from distutils.version import LooseVersion

# Build a real context in versions of Django greater than 1.6
# On versions below 1.6, create a context that simply warns that
# the query number assertion is not happening
if LooseVersion(django.get_version()) >= LooseVersion('1.6'):
    from django.test.utils import CaptureQueriesContext
    CAPTURE = True

    class _AssertNumQueriesLessThanContext(CaptureQueriesContext):
        def __init__(self, test_case, num, connection):
            self.test_case = test_case
            self.num = num
            super(_AssertNumQueriesLessThanContext, self).__init__(connection)

        def __exit__(self, exc_type, exc_value, traceback):
            super(_AssertNumQueriesLessThanContext, self).__exit__(exc_type, exc_value, traceback)
            if exc_type is not None:
                return
            executed = len(self)
            self.test_case.assertTrue(
                executed <= self.num, "%d queries executed, expected less than %d" % (
                    executed, self.num
                )
            )

else:
    CAPTURE = False

    class _AssertNumQueriesLessThanContext(object):
        def __init__(self, test_case, num, connection):
            pass

        def __enter__(self):
            pass

        def __exit__(self, exc_type, exc_value, traceback):
            warnings.warn("assertNumQueriesLessThan being skipped, does not work prior to Django 1.6")

class login(object):
    def __init__(self, testcase, **credentials):
        self.testcase = testcase
        success = testcase.client.login(**credentials)
        self.testcase.assertTrue(
            success,
            "login failed with credentials=%r" % (credentials)
        )

    def __enter__(self):
        pass

    def __exit__(self, *args):
        self.testcase.client.logout()

class TestCase(TestCase):
    """
    Django TestCase with helpful additional features
    """
    user_factory = None

    def tearDown(self):
        self.client.logout()

    def get(self, url_name, *args, **kwargs):
        """ GET url by name using reverse() """
        return self.client.get(reverse(url_name, args=args, kwargs=kwargs))

    def post(self, url_name, *args, **kwargs):
        """ POST to url by name using reverse() """
        data = kwargs.pop("data", None)
        return self.client.post(reverse(url_name, args=args, kwargs=kwargs), data)

    def response_200(self, response):
        """ Given response has status_code 200 """
        self.assertEqual(response.status_code, 200)

    def response_201(self, response):
        """ Given response has status_code 201 """
        self.assertEqual(response.status_code, 201)

    def response_302(self, response):
        """ Given response has status_code 302 """
        self.assertEqual(response.status_code, 302)

    def response_403(self, response):
        """ Given response has status_code 403 """
        self.assertEqual(response.status_code, 403)

    def response_404(self, response):
        """ Given response has status_code 404 """
        self.assertEqual(response.status_code, 404)

    def get_check_200(self, url, *args, **kwargs):
        """ Test that we can GET a page and it returns a 200 """
        response = self.get(url, *args, **kwargs)
        self.response_200(response)
        return response

    def assertLoginRequired(self, url, *args, **kwargs):
        """ Ensure login is required to GET this URL """
        res = self.get(url, *args, **kwargs)
        reversed_url = reverse(url, args=args, kwargs=kwargs)
        expected_url = "{0}?next={1}".format(settings.LOGIN_URL, reversed_url)
        self.assertRedirects(res, expected_url)

    def login(self, **credentials):
        """ Login a user """
        return login(self, **credentials)

    def reverse(self, name, *args, **kwargs):
        """ Reverse a url, convience to avoid having to import reverse in tests """
        return reverse(name, args=args, kwargs=kwargs)

    def make_user(self, username):
        """
        Build a user with <username> and password of 'password' for testing
        purposes.  Exposes the user as self.user_<username>
        """
        if self.user_factory:
            test_user = self.user_factory(username=username)
            test_user.set_password('password')
            test_user.save()
            return test_user
        else:
            if LooseVersion(django.get_version()) >= LooseVersion('1.6'):
                from django.contrib.auth import get_user_model
                User = get_user_model()
            else:
                from django.contrib.auth.models import User

            test_user = User.objects.create_user(
                username,
                '{0}@example.com'.format(username),
                'password',
            )
            return test_user

    def assertNumQueriesLessThan(self, num, func=None, *args, **kwargs):
        using = kwargs.pop("using", DEFAULT_DB_ALIAS)
        conn = connections[using]

        context = _AssertNumQueriesLessThanContext(self, num, conn)
        if func is None:
            return context

        with context:
            func(*args, **kwargs)

    def assertGoodView(self, url_name, *args, **kwargs):
        """
        Quick-n-dirty testing of a given url name.
        Ensures URL returns a 200 status and that generates less than 100
        database queries.
        """
        query_count = kwargs.pop('test_query_count', 100)

        with self.assertNumQueriesLessThan(query_count):
            response = self.get(url_name, *args, **kwargs)

        self.response_200(response)

<|endoftext|>


<commit_before>import json
import numpy as np
from IPython import embed
import os
from collections import OrderedDict
import pandas as pd
from warnings import warn

def sigm_tf(x):
    return 1./(1 + np.exp(-1 * x))

def sigm(x):
    return 2./(1 + np.exp(-2 * x)) - 1

class QuaLiKizMultiNN():
    def __init__(self, nns):
        self._nns = nns
        feature_names = nns[0]
        for nn in self._nns:
            if len(nn.target_names) == 1:
                name = nn.target_names[0]
            else:
                NotImplementedError('Multitarget not implemented yet')
            if np.all(nn.feature_names.ne(feature_names)):
                Exception('Supplied NNs have different feature names')
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')

    @property
    def target_names(self):
        targets = []
        for nn in self._nns:
            targets.extend(list(nn.target_names))
        return targets

    def get_output(self, target, **kwargs):
        for nn in self._nns:
            if [target] == list(nn.target_names):
                result = nn.get_output(**kwargs)
                break
            elif target in nn.target_names.values:
                NotImplementedError('Multitarget not implemented yet')
        return result

    def get_outputs(self, **kwargs):
        results = pd.DataFrame()
        feature_max = -np.inf
        feature_min = np.inf
        for nn in self._nns:
            if len(nn.target_names) == 1:
                name = nn.target_names[0]
                results[name] = nn.get_output(**kwargs)
            elif target in nn.target_names.values:
                NotImplementedError('Multitarget not implemented yet')

        return results

    @property
    def target_names(self):
        target_names = []
        for nn in self._nns:
            target_names.extend(nn.target_names)
        return list(set(target_names))

    @property
    def feature_names(self):
        return self._nns[0].feature_names

    @property
    def feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0].feature_max, np.inf),
                                index=self._nns[0].feature_max.index)
        for nn in self._nns:
            feature_max = nn.feature_max.combine(feature_max, min)
        return feature_max

    @property
    def feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0].feature_min, -np.inf),
                                index=self._nns[0].feature_min.index)
        for nn in self._nns:
            feature_min = nn.feature_min.combine(feature_min, max)
        return feature_min

class QuaLiKizComboNN():
    def __init__(self, target_name, nns, combo_func):
        self._nns = nns
        feature_names = nns[0]
        for nn in self._nns:
            if np.all(nn.feature_names.ne(feature_names)):
                Exception('Supplied NNs have different feature names')
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')

        self._combo_func = combo_func
        self._target_name = target_name

    def get_output(self, **kwargs):
        output = pd.DataFrame()
        #output1 = self._nn1.get_output(**kwargs).values
        #output2 = self._nn2.get_output(**kwargs).values
        #output[self._target_name] = np.squeeze(self._combo_func(output1, output2))
        output[self._target_name] = np.squeeze(self._combo_func(*[nn.get_output(**kwargs).as_matrix() for nn in self._nns]))
        return output

    @property
    def target_names(self):
        return [self._target_name]

    @property
    def feature_names(self):
        return self._nns[0].feature_names

    @property
    def feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0].feature_max, np.inf),
                                index=self._nns[0].feature_max.index)
        for nn in self._nns:
            feature_max = nn.feature_max.combine(feature_max, min)
        return feature_max

    @property
    def feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0].feature_min, -np.inf),
                                index=self._nns[0].feature_min.index)
        for nn in self._nns:
            feature_min = nn.feature_min.combine(feature_min, max)
        return feature_min

class QuaLiKizDuoNN():
    def __init__(self, target_name, nn1, nn2, combo_func):
        self._nn1 = nn1
        self._nn2 = nn2
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')
        if np.all(nn1.feature_names.ne(nn2.feature_names)):
            Exception('Supplied NNs have different feature names')
        self._combo_func = combo_func
        self._target_name = target_name

    def get_output(self, **kwargs):
        output = pd.DataFrame()
        output1 = self._nn1.get_output(**kwargs).values
        output2 = self._nn2.get_output(**kwargs).values
        output[self._target_name] = np.squeeze(self._combo_func(output1, output2))
        return output

    @property
    def target_names(self):
        return [self._target_name]

    @property
    def feature_names(self):
        return self._nn1.feature_names

    @property
    def feature_max(self):
        return self._nn1.feature_max.combine(self._nn2.feature_max, min)

    @property
    def feature_min(self):
        return self._nn1.feature_min.combine(self._nn2.feature_min, max)

class QuaLiKizNDNN():
    def __init__(self, nn_dict):
        """ General ND fully-connected multilayer perceptron neural network

        Initialize this class using a nn_dict. This dict is usually read
        directly from JSON, and has a specific structure. Generate this JSON
        file using the supplied function in QuaLiKiz-Tensorflow
        """
        parsed = {}
        # Read and parse the json. E.g. put arrays in arrays and the rest in a dict
        for name, value in nn_dict.items():
            if value.__class__ == list:
                parsed[name] = np.array(value)
            else:
                parsed[name] = dict(value)
        # These variables do not depend on the amount of layers in the NN
        for name in ['prescale_bias', 'prescale_factor', 'feature_min',
                     'feature_max', 'feature_names', 'target_names']:
            setattr(self, name, pd.Series(parsed.pop(name), name=name))
        self.layers = []
        # Now find out the amount of layers in our NN, and save the weigths and biases
        for ii in range(1, len(parsed)+1):
            try:
                name = 'layer' + str(ii)
                weight = parsed.pop(name + '/weights/Variable:0')
                bias = parsed.pop(name + '/biases/Variable:0')
                activation = sigm
                self.layers.append(QuaLiKizNDNN.NNLayer(weight, bias, activation))
            except KeyError:
                # This name does not exist in the JSON,
                # so our previously read layer was the one
                break
        # Ignore metadata
        try:
            del parsed['_metadata']
        except KeyError:
            pass
        assert not any(parsed), 'nn_dict not fully parsed! ' + str(parsed)

    def apply_layers(self, input):
        """ Apply all NN layers to the given input

        The given input has to be array-like, but can be of size 1
        """
        for layer in self.layers:
            input = layer.apply(input)
        return input

    class NNLayer():
        """ A single (hidden) NN layer
        A hidden NN layer is just does

        output = activation(weight * input + bias)

        Where weight is generally a matrix; output, input and bias a vector
        and activation a (sigmoid) function.
        """
        def __init__(self, weight, bias, activation):
            self.weight = weight
            self.bias = bias
            self.activation = activation

        def apply(self, input):
            preactivation = np.matmul(input, self.weight) + self.bias
            result = self.activation(preactivation)
            return result

        def shape(self):
            return self.weight.shape

        def __str__(self):
            return ('NNLayer shape ' + str(self.shape()))

    def get_output(self, **kwargs):
        """ Calculate the output given a specific input

        This function accepts inputs in the form of a dict with
        as keys the name of the specific input variable (usually
        at least the feature_names) and as values 1xN same-length
        arrays.
        """
        nn_input = pd.DataFrame()
        # Read and scale the inputs
        for name in self.feature_names:
            try:
                value = kwargs.pop(name)
                nn_input[name] = self.prescale_factor[name] * value + self.prescale_bias[name]
            except KeyError as e:
                raise Exception('NN needs \'' + name + '\' as input')

        output = pd.DataFrame()
        # Apply all NN layers an re-scale the outputs
        for name in self.target_names:
            nn_output = (np.squeeze(self.apply_layers(nn_input)) - self.prescale_bias[name]) / self.prescale_factor[name]
            output[name] = nn_output

        if any(kwargs):
            for name in kwargs:
                warn('input dict not fully parsed! Did not use ' + name)
        return output

    @classmethod
    def from_json(cls, json_file):
        with open(json_file) as file_:
            dict_ = json.load(file_)
        nn = QuaLiKizNDNN(dict_)
        return nn

if __name__ == '__main__':
    # Test the function
    root = os.path.dirname(os.path.realpath(__file__))
    nn1 = QuaLiKizNDNN.from_json(os.path.join(root, 'nn_efe_GB.json'))
    nn2 = QuaLiKizNDNN.from_json(os.path.join(root, 'nn_efi_GB.json'))
    nn3 = QuaLiKizDuoNN('nn_eftot_GB', nn1, nn2, lambda x, y: x + y)
    nn = QuaLiKizMultiNN([nn1, nn2])
    scann = 24
    input = pd.DataFrame()
    input['Ati'] = np.array(np.linspace(2,13, scann))
    input['Ti_Te']  = np.full_like(input['Ati'], 1.)
    input['Zeffx']  = np.full_like(input['Ati'], 1.)
    input['An']  = np.full_like(input['Ati'], 2.)
    input['Ate']  = np.full_like(input['Ati'], 5.)
    input['qx'] = np.full_like(input['Ati'], 0.660156)
    input['smag']  = np.full_like(input['Ati'], 0.399902)
    input['Nustar']  = np.full_like(input['Ati'], 0.009995)
    input['x']  = np.full_like(input['Ati'], 0.449951)
    fluxes = nn.get_outputs(**input)
    print(fluxes)
    embed()
<commit_msg>Added NN clipping (based on target_max and target_min)<commit_after>import json
import numpy as np
from IPython import embed
import os
from collections import OrderedDict
import pandas as pd
from warnings import warn

def sigm_tf(x):
    return 1./(1 + np.exp(-1 * x))

def sigm(x):
    return 2./(1 + np.exp(-2 * x)) - 1

class QuaLiKizMultiNN():
    def __init__(self, nns):
        self._nns = nns
        feature_names = nns[0]
        for nn in self._nns:
            if len(nn.target_names) == 1:
                name = nn.target_names[0]
            else:
                NotImplementedError('Multitarget not implemented yet')
            if np.all(nn.feature_names.ne(feature_names)):
                Exception('Supplied NNs have different feature names')
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')

    @property
    def target_names(self):
        targets = []
        for nn in self._nns:
            targets.extend(list(nn.target_names))
        return targets

    def get_output(self, target, **kwargs):
        for nn in self._nns:
            if [target] == list(nn.target_names):
                result = nn.get_output(**kwargs)
                break
            elif target in nn.target_names.values:
                NotImplementedError('Multitarget not implemented yet')
        return result

    def get_outputs(self, **kwargs):
        results = pd.DataFrame()
        feature_max = -np.inf
        feature_min = np.inf
        for nn in self._nns:
            if len(nn.target_names) == 1:
                name = nn.target_names[0]
                results[name] = nn.get_output(**kwargs)
            elif target in nn.target_names.values:
                NotImplementedError('Multitarget not implemented yet')

        return results

    @property
    def target_names(self):
        target_names = []
        for nn in self._nns:
            target_names.extend(nn.target_names)
        return list(set(target_names))

    @property
    def feature_names(self):
        return self._nns[0].feature_names

    @property
    def feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0].feature_max, np.inf),
                                index=self._nns[0].feature_max.index)
        for nn in self._nns:
            feature_max = nn.feature_max.combine(feature_max, min)
        return feature_max

    @property
    def feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0].feature_min, -np.inf),
                                index=self._nns[0].feature_min.index)
        for nn in self._nns:
            feature_min = nn.feature_min.combine(feature_min, max)
        return feature_min

class QuaLiKizComboNN():
    def __init__(self, target_name, nns, combo_func):
        self._nns = nns
        feature_names = nns[0]
        for nn in self._nns:
            if np.all(nn.feature_names.ne(feature_names)):
                Exception('Supplied NNs have different feature names')
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')

        self._combo_func = combo_func
        self._target_name = target_name

    def get_output(self, **kwargs):
        output = pd.DataFrame()
        #output1 = self._nn1.get_output(**kwargs).values
        #output2 = self._nn2.get_output(**kwargs).values
        #output[self._target_name] = np.squeeze(self._combo_func(output1, output2))
        output[self._target_name] = np.squeeze(self._combo_func(*[nn.get_output(**kwargs).as_matrix() for nn in self._nns]))
        return output

    @property
    def target_names(self):
        return [self._target_name]

    @property
    def feature_names(self):
        return self._nns[0].feature_names

    @property
    def feature_max(self):
        feature_max = pd.Series(np.full_like(self._nns[0].feature_max, np.inf),
                                index=self._nns[0].feature_max.index)
        for nn in self._nns:
            feature_max = nn.feature_max.combine(feature_max, min)
        return feature_max

    @property
    def feature_min(self):
        feature_min = pd.Series(np.full_like(self._nns[0].feature_min, -np.inf),
                                index=self._nns[0].feature_min.index)
        for nn in self._nns:
            feature_min = nn.feature_min.combine(feature_min, max)
        return feature_min

class QuaLiKizDuoNN():
    def __init__(self, target_name, nn1, nn2, combo_func):
        self._nn1 = nn1
        self._nn2 = nn2
        if np.any(self.feature_min > self.feature_max):
            raise Exception('Feature min > feature max')
        if np.all(nn1.feature_names.ne(nn2.feature_names)):
            Exception('Supplied NNs have different feature names')
        self._combo_func = combo_func
        self._target_name = target_name

    def get_output(self, **kwargs):
        output = pd.DataFrame()
        output1 = self._nn1.get_output(**kwargs).values
        output2 = self._nn2.get_output(**kwargs).values
        output[self._target_name] = np.squeeze(self._combo_func(output1, output2))
        return output

    @property
    def target_names(self):
        return [self._target_name]

    @property
    def feature_names(self):
        return self._nn1.feature_names

    @property
    def feature_max(self):
        return self._nn1.feature_max.combine(self._nn2.feature_max, min)

    @property
    def feature_min(self):
        return self._nn1.feature_min.combine(self._nn2.feature_min, max)

class QuaLiKizNDNN():
    def __init__(self, nn_dict):
        """ General ND fully-connected multilayer perceptron neural network

        Initialize this class using a nn_dict. This dict is usually read
        directly from JSON, and has a specific structure. Generate this JSON
        file using the supplied function in QuaLiKiz-Tensorflow
        """
        parsed = {}
        # Read and parse the json. E.g. put arrays in arrays and the rest in a dict
        for name, value in nn_dict.items():
            if value.__class__ == list:
                parsed[name] = np.array(value)
            else:
                parsed[name] = dict(value)
        # These variables do not depend on the amount of layers in the NN
        for name in ['prescale_bias', 'prescale_factor', 'feature_min',
                     'feature_max', 'feature_names', 'target_names', 'target_min', 'target_max']:
            setattr(self, name, pd.Series(parsed.pop(name), name=name))
        self.layers = []
        # Now find out the amount of layers in our NN, and save the weigths and biases
        for ii in range(1, len(parsed)+1):
            try:
                name = 'layer' + str(ii)
                weight = parsed.pop(name + '/weights/Variable:0')
                bias = parsed.pop(name + '/biases/Variable:0')
                activation = sigm
                self.layers.append(QuaLiKizNDNN.NNLayer(weight, bias, activation))
            except KeyError:
                # This name does not exist in the JSON,
                # so our previously read layer was the one
                break
        try:
            self._clip_zeros = parsed['_metadata']['clip_bounds']
        except KeyError:
            self._clip_zeros = False
        # Ignore metadata
        try:
            del parsed['_metadata']
        except KeyError:
            pass
        if any(parsed):
            warn('nn_dict not fully parsed! ' + str(parsed))

    def apply_layers(self, input):
        """ Apply all NN layers to the given input

        The given input has to be array-like, but can be of size 1
        """
        for layer in self.layers:
            input = layer.apply(input)
        return input

    class NNLayer():
        """ A single (hidden) NN layer
        A hidden NN layer is just does

        output = activation(weight * input + bias)

        Where weight is generally a matrix; output, input and bias a vector
        and activation a (sigmoid) function.
        """
        def __init__(self, weight, bias, activation):
            self.weight = weight
            self.bias = bias
            self.activation = activation

        def apply(self, input):
            preactivation = np.dot(input, self.weight) + self.bias
            result = self.activation(preactivation)
            return result

        def shape(self):
            return self.weight.shape

        def __str__(self):
            return ('NNLayer shape ' + str(self.shape()))

    def get_output(self, clip_data=True, **kwargs):
        """ Calculate the output given a specific input

        This function accepts inputs in the form of a dict with
        as keys the name of the specific input variable (usually
        at least the feature_names) and as values 1xN same-length
        arrays.
        """
        nn_input = pd.DataFrame()
        # Read and scale the inputs
        for name in self.feature_names:
            try:
                value = kwargs.pop(name)
                nn_input[name] = self.prescale_factor[name] * value + self.prescale_bias[name]
            except KeyError as e:
                raise Exception('NN needs \'' + name + '\' as input')

        output = pd.DataFrame()
        # Apply all NN layers an re-scale the outputs
        for name in self.target_names:
            nn_output = (np.squeeze(self.apply_layers(nn_input)) - self.prescale_bias[name]) / self.prescale_factor[name]
            output[name] = nn_output

        if clip_data:
            for name, column in output.items():
                output[output < self.target_min[name]] = self.target_min[name]
                output[output > self.target_max[name]] = self.target_max[name]

        if any(kwargs):
            for name in kwargs:
                warn('input dict not fully parsed! Did not use ' + name)
        return output

    @classmethod
    def from_json(cls, json_file):
        with open(json_file) as file_:
            dict_ = json.load(file_)
        nn = QuaLiKizNDNN(dict_)
        return nn

if __name__ == '__main__':
    # Test the function
    root = os.path.dirname(os.path.realpath(__file__))
    nn1 = QuaLiKizNDNN.from_json(os.path.join(root, 'nn_efe_GB.json'))
    nn2 = QuaLiKizNDNN.from_json(os.path.join(root, 'nn_efi_GB.json'))
    nn3 = QuaLiKizDuoNN('nn_eftot_GB', nn1, nn2, lambda x, y: x + y)
    nn = QuaLiKizMultiNN([nn1, nn2])
    scann = 24
    input = pd.DataFrame()
    input['Ati'] = np.array(np.linspace(2,13, scann))
    input['Ti_Te']  = np.full_like(input['Ati'], 1.)
    input['Zeffx']  = np.full_like(input['Ati'], 1.)
    input['An']  = np.full_like(input['Ati'], 2.)
    input['Ate']  = np.full_like(input['Ati'], 5.)
    input['qx'] = np.full_like(input['Ati'], 0.660156)
    input['smag']  = np.full_like(input['Ati'], 0.399902)
    input['Nustar']  = np.full_like(input['Ati'], 0.009995)
    input['x']  = np.full_like(input['Ati'], 0.449951)
    fluxes = nn.get_outputs(**input)
    print(fluxes)
    embed()
<|endoftext|>


<commit_before>
import discord
from discord.ext import commands

import asyncio
import contextlib

import pydealer
import treys

from utilities import checks

def setup(bot):
	bot.add_cog(Poker())

class Poker(commands.Cog):

	def __init__(self):
		self.poker_hands = {}

	@commands.group(invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def poker(self, ctx):
		'''Texas Hold'em'''
		if not (poker_hand := self.poker_hands.get(ctx.channel.id)):
			self.poker_hands[ctx.channel.id] = PokerHand()
			await self.poker_hands[ctx.channel.id].initiate(ctx)
		elif poker_hand.started:
			await ctx.embed_reply(f"{ctx.bot.error_emoji} There's already a poker match in progress here")
		elif ctx.author not in poker_hand.hands:
			await poker_hand.add_player(ctx)
		elif len(poker_hand.hands) == 1:
			await ctx.embed_reply("Unfortunately, you can't play poker alone")
			# TODO: Allow playing against bot
		else:
			await poker_hand.start(ctx)
			del self.poker_hands[ctx.channel.id]

def cards_to_string(cards):
	return " | ".join(f":{card.suit.lower()}: {card.value}" for card in cards)

class PokerHand:

	def __init__(self):
		self.deck = pydealer.Deck()
		self.deck.shuffle()
		self.community_cards = self.deck.deal(5)
		self.pot = 0
		self.started = False

	async def initiate(self, ctx):
		self.hands = {ctx.author: self.deck.deal(2)}
		self.initial_message = await ctx.embed_reply(f"{ctx.author.mention} is initiating a poker match\n\n"
														f"`{ctx.prefix}poker` to join\n"
														f"`{ctx.prefix}poker` again to start",
														author_name = discord.Embed.Empty)

	async def add_player(self, ctx):
		self.hands[ctx.author] = self.deck.deal(2)
		embed = self.initial_message.embeds[0]
		index = embed.description.find('\n\n') + 1
		embed.description = embed.description[:index] + f"{ctx.author.mention} has joined\n" + embed.description[index:]
		await self.initial_message.edit(embed = embed)
		await ctx.bot.attempt_delete_message(ctx.message)

	async def start(self, ctx):
		self.started = True

		embed = self.initial_message.embeds[0]
		index = embed.description.find('\n\n') + 1
		embed.description = embed.description[:index] + f"{ctx.author.mention} has started the match"
		await self.initial_message.edit(embed = embed)
		await ctx.bot.attempt_delete_message(ctx.message)

		for player, hand in self.hands.items():
			await ctx.bot.send_embed(player, f"Your poker hand: {cards_to_string(hand.cards)}")

		round_message = None
		for stage, number_of_cards in zip(("flop", "turn", "river", "showdown"), (3, 4, 5, 5)):
			await self.betting(ctx, round_message)
			if len(self.hands) == 1:
				return await ctx.embed_send(f"{next(iter(self.hands.keys())).mention} is the winner of {self.pot}")
			round_message = await ctx.embed_send(f"The pot: {self.pot}\n"
													f"The {stage}: {cards_to_string(self.community_cards[:number_of_cards])}")

		evaluator = treys.Evaluator()
		board = []
		for card in self.community_cards.cards:
			abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
			board.append(treys.Card.new(abbreviation))
		best_hand_value = evaluator.table.MAX_HIGH_CARD
		for player, hand in self.hands.items():
			hand_stack = []
			for card in hand:
				abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
				hand_stack.append(treys.Card.new(abbreviation))
			value = evaluator.evaluate(board, hand_stack)
			if value < best_hand_value:
				best_hand_value = value
				winner = player

		hand_name = evaluator.class_to_string(evaluator.get_rank_class(best_hand_value))
		embed = round_message.embeds[0]
		embed.description += (f"\n\n{winner.mention} is the winner of {self.pot} with a {hand_name}\n"
								f"{winner.mention}'s hand: {cards_to_string(self.hands.pop(winner))}\n")
		for player, hand in self.hands.items():
			embed.description += f"\n{player.mention}'s hand: {cards_to_string(hand)}"
			# TODO: Allow mucking
		await round_message.edit(embed = embed)

	async def betting(self, ctx, message = None):
		bets = {}
		current_bet = 0
		first_player_turn = True
		while (not bets and len(self.hands) != 1) or not all(bet == current_bet for bet in bets.values()):
			for player in self.hands.copy():
				def check(message):
					if message.author != player:
						return False
					if message.content.lower() in ("call", "check", "fold"):
						return True
					if message.content.lower().startswith("raise "):
						try:
							return int(message.content[6:]) >= 0  # Use .removeprefix in Python 3.9
						except ValueError:
							return False
					return False
				can_check = bets.get(player, 0) == current_bet
				turn_message = (f"{player.mention}'s turn:\n"
								f"Would you like to `{'check' if can_check else 'call'}`, `fold`, or `raise ` an amount?")
				if not can_check:
					turn_message += f"\n{current_bet - bets.get(player, 0)} to `call`"
				if not message:
					initial_embed = discord.Embed(color = ctx.bot.bot_color)
					initial_embed.description = ""
					message = await ctx.embed_send(turn_message)
					embed = message.embeds[0]
					first_player_turn = False
				else:
					embed = message.embeds[0]
					initial_embed = embed.copy()
					if first_player_turn:
						initial_embed.description += '\n'
						first_player_turn = False
					embed.description += "\n\n" + turn_message
					await message.edit(embed = embed)
				while (response := await ctx.bot.wait_for("message", check = check)).content.lower() == "check" and not can_check:
					embed_copy = embed.copy()
					embed_copy.description += f"\n{player.mention} attempted to check, but there is a bet to call"
					await message.edit(embed = embed_copy)
					await ctx.bot.attempt_delete_message(response)
				if response.content.lower() == "call":
					if can_check:
						initial_embed.description += f"\n{player.mention} has checked"
					else:
						initial_embed.description += f"\n{player.mention} has called"
					bets[player] = current_bet
				elif response.content.lower() == "check":
					initial_embed.description += f"\n{player.mention} has checked"
					bets[player] = current_bet
				elif response.content.lower() == "fold":
					self.pot += bets.pop(player, 0)
					hand = self.hands.pop(player)
					initial_embed.description += f"\n{player.mention} has folded"
					prompt = await ctx.embed_send(f"\n{player.mention}: Would you like to show your hand?")
					with contextlib.suppress(asyncio.TimeoutError):
						if (await ctx.bot.wait_for_yes_or_no(channel = ctx.channel, message = prompt, user = player,
																timeout = 10, use_reactions = True)):
							initial_embed.description += f"\n{player.mention}'s hand was {cards_to_string(hand)}"
					await ctx.bot.attempt_delete_message(prompt)
				elif response.content.lower().startswith("raise "):
					amount = int(response.content[6:])  # Use .removeprefix in Python 3.9
					if not amount:
						if can_check:
							initial_embed.description += f"\n{player.mention} has checked"
						else:
							initial_embed.description += f"\n{player.mention} has called"
					else:
						current_bet += amount
						initial_embed.description += f"\n{player.mention} has raised to {current_bet}"
					bets[player] = current_bet
				await message.edit(embed = initial_embed)
				await ctx.bot.attempt_delete_message(response)
				if len(self.hands) == 1 or (len(self.hands) == len(bets) and all(bet == current_bet for bet in bets.values())):
					break
		self.pot += sum(bets.values())

<commit_msg>[Discord] Improve clarity and formatting of PokerHand betting method<commit_after>
import discord
from discord.ext import commands

import asyncio
import contextlib

import pydealer
import treys

from utilities import checks

def setup(bot):
	bot.add_cog(Poker())

class Poker(commands.Cog):

	def __init__(self):
		self.poker_hands = {}

	@commands.group(invoke_without_command = True, case_insensitive = True)
	@checks.not_forbidden()
	async def poker(self, ctx):
		'''Texas Hold'em'''
		if not (poker_hand := self.poker_hands.get(ctx.channel.id)):
			self.poker_hands[ctx.channel.id] = PokerHand()
			await self.poker_hands[ctx.channel.id].initiate(ctx)
		elif poker_hand.started:
			await ctx.embed_reply(f"{ctx.bot.error_emoji} There's already a poker match in progress here")
		elif ctx.author not in poker_hand.hands:
			await poker_hand.add_player(ctx)
		elif len(poker_hand.hands) == 1:
			await ctx.embed_reply("Unfortunately, you can't play poker alone")
			# TODO: Allow playing against bot
		else:
			await poker_hand.start(ctx)
			del self.poker_hands[ctx.channel.id]

def cards_to_string(cards):
	return " | ".join(f":{card.suit.lower()}: {card.value}" for card in cards)

class PokerHand:

	def __init__(self):
		self.deck = pydealer.Deck()
		self.deck.shuffle()
		self.community_cards = self.deck.deal(5)
		self.pot = 0
		self.started = False

	async def initiate(self, ctx):
		self.hands = {ctx.author: self.deck.deal(2)}
		self.initial_message = await ctx.embed_reply(f"{ctx.author.mention} is initiating a poker match\n\n"
														f"`{ctx.prefix}poker` to join\n"
														f"`{ctx.prefix}poker` again to start",
														author_name = discord.Embed.Empty)

	async def add_player(self, ctx):
		self.hands[ctx.author] = self.deck.deal(2)
		embed = self.initial_message.embeds[0]
		index = embed.description.find('\n\n') + 1
		embed.description = embed.description[:index] + f"{ctx.author.mention} has joined\n" + embed.description[index:]
		await self.initial_message.edit(embed = embed)
		await ctx.bot.attempt_delete_message(ctx.message)

	async def start(self, ctx):
		self.started = True

		embed = self.initial_message.embeds[0]
		index = embed.description.find('\n\n') + 1
		embed.description = embed.description[:index] + f"{ctx.author.mention} has started the match"
		await self.initial_message.edit(embed = embed)
		await ctx.bot.attempt_delete_message(ctx.message)

		for player, hand in self.hands.items():
			await ctx.bot.send_embed(player, f"Your poker hand: {cards_to_string(hand.cards)}")

		round_message = None
		for stage, number_of_cards in zip(("flop", "turn", "river", "showdown"), (3, 4, 5, 5)):
			await self.betting(ctx, round_message)
			if len(self.hands) == 1:
				return await ctx.embed_send(f"{next(iter(self.hands.keys())).mention} is the winner of {self.pot}")
			round_message = await ctx.embed_send(f"The pot: {self.pot}\n"
													f"The {stage}: {cards_to_string(self.community_cards[:number_of_cards])}")

		evaluator = treys.Evaluator()
		board = []
		for card in self.community_cards.cards:
			abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
			board.append(treys.Card.new(abbreviation))
		best_hand_value = evaluator.table.MAX_HIGH_CARD
		for player, hand in self.hands.items():
			hand_stack = []
			for card in hand:
				abbreviation = pydealer.card.card_abbrev(card.value[0] if card.value != "10" else 'T', card.suit[0].lower())
				hand_stack.append(treys.Card.new(abbreviation))
			value = evaluator.evaluate(board, hand_stack)
			if value < best_hand_value:
				best_hand_value = value
				winner = player

		hand_name = evaluator.class_to_string(evaluator.get_rank_class(best_hand_value))
		embed = round_message.embeds[0]
		embed.description += (f"\n\n{winner.mention} is the winner of {self.pot} with a {hand_name}\n"
								f"{winner.mention}'s hand: {cards_to_string(self.hands.pop(winner))}\n")
		for player, hand in self.hands.items():
			embed.description += f"\n{player.mention}'s hand: {cards_to_string(hand)}"
			# TODO: Allow mucking
		await round_message.edit(embed = embed)

	async def betting(self, ctx, message = None):
		bets = {}
		current_bet = 0
		first_player_turn = True
		while (not bets and len(self.hands) != 1) or not all(bet == current_bet for bet in bets.values()):
			for player in self.hands.copy():
				can_check = bets.get(player, 0) == current_bet
				turn_message = (f"{player.mention}'s turn:\n"
								f"Would you like to `{'check' if can_check else 'call'}`, `fold`, or `raise ` an amount?")
				if not can_check:
					turn_message += f"\n{current_bet - bets.get(player, 0)} to `call`"
				if not message:
					initial_embed = discord.Embed(color = ctx.bot.bot_color)
					initial_embed.description = ""
					message = await ctx.embed_send(turn_message)
					embed = message.embeds[0]
					first_player_turn = False
				else:
					embed = message.embeds[0]
					initial_embed = embed.copy()
					if first_player_turn:
						initial_embed.description += '\n'
						first_player_turn = False
					embed.description += "\n\n" + turn_message
					await message.edit(embed = embed)

				def check(message):
					if message.author != player:
						return False
					if message.content.lower() in ("call", "check", "fold"):
						return True
					if message.content.lower().startswith("raise "):
						try:
							return int(message.content[6:]) >= 0  # Use .removeprefix in Python 3.9
						except ValueError:
							return False
					return False

				while (response := await ctx.bot.wait_for("message", check = check)).content.lower() == "check" and not can_check:
					embed_copy = embed.copy()
					embed_copy.description += f"\n{player.mention} attempted to check, but there is a bet to call"
					await message.edit(embed = embed_copy)
					await ctx.bot.attempt_delete_message(response)
				if response.content.lower() == "call":
					if can_check:
						initial_embed.description += f"\n{player.mention} has checked"
					else:
						initial_embed.description += f"\n{player.mention} has called"
					bets[player] = current_bet
				elif response.content.lower() == "check":
					initial_embed.description += f"\n{player.mention} has checked"
					bets[player] = current_bet
				elif response.content.lower() == "fold":
					self.pot += bets.pop(player, 0)
					hand = self.hands.pop(player)
					initial_embed.description += f"\n{player.mention} has folded"
					prompt = await ctx.embed_send(f"\n{player.mention}: Would you like to show your hand?")
					with contextlib.suppress(asyncio.TimeoutError):
						if (await ctx.bot.wait_for_yes_or_no(channel = ctx.channel, message = prompt, user = player,
																timeout = 10, use_reactions = True)):
							initial_embed.description += f"\n{player.mention}'s hand was {cards_to_string(hand)}"
					await ctx.bot.attempt_delete_message(prompt)
				elif response.content.lower().startswith("raise "):
					amount = int(response.content[6:])  # Use .removeprefix in Python 3.9
					if not amount:
						if can_check:
							initial_embed.description += f"\n{player.mention} has checked"
						else:
							initial_embed.description += f"\n{player.mention} has called"
					else:
						current_bet += amount
						initial_embed.description += f"\n{player.mention} has raised to {current_bet}"
					bets[player] = current_bet
				await message.edit(embed = initial_embed)
				await ctx.bot.attempt_delete_message(response)

				if len(self.hands) == 1 or (len(self.hands) == len(bets) and all(bet == current_bet for bet in bets.values())):
					break
		self.pot += sum(bets.values())

<|endoftext|>