<commit_before>#!/usr/bin/env python

import sys
import re
import os
from subprocess import Popen, PIPE
import string

SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
repeatThreashold = 4
tmp_settings_file = '/tmp/TMP.gem_settings.nml'

# Get namelist
nlfile=''
if len(sys.argv) >= 2:
    nlfile=sys.argv[1]

# First, remove some of the known replacement strings
# Often the 'aftermod' file should be used (which doesn't include these)
# rather than the 'beforemod' fileo
#
# In any case, skip this for now.
if (False):
    f=open(nlfile,'r')
    output='';
    for line in f.readlines():
        # Start and end dates
        line = re.sub(r"((strt_|end_).*)\s*=\s*('|\")[a-zA-Z].*('|\")\s*,", r"\1 = '0',", line)

        # Other known constants
        line = re.sub(r"=(\s*)PTOPO_NPE.", r"=\1 0        ", line)

        output = output + line;
    f = open(tmp_settings_file,'w')
    f.write(output)
    f.close()

    nlfile=tmp_settings_file

proc = Popen("%s/_readNamelist %s"%(SCRIPT_PATH, nlfile), stdout=PIPE, stderr=None, shell=True)
(namelistContent, err) = proc.communicate()

it = iter(namelistContent.splitlines())
#print it.next()

repeat = 0;
lastline = "";
namelist=''
output=''
reKey = re.compile(r'[a-zA-Z].*=')
for i,line in enumerate(it):
    # Figure out what namelist we're in
    match = re.search(r'\&(\w+)', line)
    if match:
        namelist=match.group(1)

    # Ignore lines outside a namelist
    #if namelist == '':
    #	continue;

    # Clean up input a bit
    line = re.sub(r'^\s([a-zA-Z])', r'\1', line, re.IGNORECASE)
    line = re.sub(r'\s+,', r',', line)
    line = re.sub(r'\s*=\s+', r' = ', line)
    line = re.sub(r"^\s*([\d'-])", r' \1', line, re.IGNORECASE)
    line = re.sub(r"\s*$", r'', line, re.IGNORECASE)
    line = re.sub(r"\s*&", r'&', line, re.IGNORECASE)

    # Get rid of messed up characters
    line = filter(lambda x: x in string.printable, line)

    # Get rid of blank/useless lines
    if line == ",":
        continue;

    # Check for repeats
    if line == lastline:
        repeat+=1;
        if repeat>repeatThreashold:
            continue;
    else:
        if repeat>repeatThreashold:
            # This adds an annoying coupling to the contactenation section below,
            # only real clean way to avoid this would be to add another loup
            # explicitly for concatenation
            line = " <Line repeated %d times>"%(repeat-repeatThreashold) + \
                    ('\n' if reKey.match(line, re.IGNORECASE) else ',') + line
        repeat = 0;

    # Now, concatenate arrays into the same line.  Hopefully this'll still work with gfortran
    matchNN = re.search(r'^\s*&', line, re.IGNORECASE)
    matchArray = re.search(r'^\s+(?=.+=.+)', line)
    if reKey.match(line, re.IGNORECASE) or matchNN or line==' /':
        #if match or matchNN or line==' /':
        output += '\n'
        #print 'reKey=',reKey.match(line, re.IGNORECASE),' match=',reKey.search(line, re.IGNORECASE), ', re.match=', match
        if matchNN:
            output += '\n'
        #output += "%3d: "%(i)
    elif matchArray:
        line = re.sub(r'^\s*(?= .*=)', r'', line)
    output += line;
    lastline=line

# Now, comform this to CSV, keeping the values in a second column
# (even if they're arrays)
oldOutput=output;
output='';
reKeyVal=re.compile(r"(\w[^ ]*)\s*=\s*(.*)$");
it = iter(oldOutput.splitlines())
for i,line in enumerate(it):
    # Remove ending comma, I know it's in the original, but it's ugly
    line = re.sub(',$', '', line)

    # Normal key/values
    if reKeyVal.match(line):
        output += reKeyVal.sub(r'"\1","\2"', line, re.IGNORECASE) + '\n'
    else:
        output += '"%s",""\n'%(line)

    #output += line + '\n'

print output

if os.path.isfile(tmp_settings_file):
    os.unlink(tmp_settings_file)
<commit_msg>Refactored everything into a class<commit_after>#!/usr/bin/env python
import sys
import re
import os
from subprocess import Popen, PIPE
import string

class NSParser(object):
    """ class to make dealing with parsing easier """
    repeat_threshold = 4
    re_key = re.compile(r'[a-zA-Z].*=')

    def __init__(self, proc):
        (namelist_content, err) = proc.communicate()
        self.it = iter(namelist_content.splitlines())

        self.repeat = 0
        self.output = ''

    @staticmethod
    def strip_whitespace(line):
        """ strips unecessary whitespace *including trailing newlines* from a single line """
        line = re.sub(r'^\s([a-zA-Z])', r'\1', line, re.IGNORECASE)
        line = re.sub(r'\s+,', r',', line)
        line = re.sub(r'\s*=\s+', r' = ', line)
        line = re.sub(r"^\s*([\d'-])", r' \1', line, re.IGNORECASE)
        line = re.sub(r"\s*$", r'', line, re.IGNORECASE)
        line = re.sub(r"\s*&", r'&', line, re.IGNORECASE)
        return line

    @staticmethod
    def filter_line(line):
        """ filters a single line """
        line = filter(lambda x: x in string.printable, line)
        return line

    def parse_line(self, line, lastline):
        """ parses a single line """
        result = ''

        if line == lastline:
            self.repeat += 1
            if self.repeat > NSParser.repeat_threshold:
                return
        else:
            if self.repeat > NSParser.repeat_threshold:
                line = " <Line repeated %d times>" % ( self.repeat - NSParser.repeat_threshold ) + ('\n' if NSParser.re_key.match(line, re.IGNORECASE) else ',') + line
            self.repeat = 0

        # Now, concatenate arrays into the same line.
        # Hopefully this'll still work with gfortran
        match_nn = re.search(r'^\s*&', line, re.IGNORECASE)
        match_array = re.search(r'^\s+(?=.+=.+)', line)

        if NSParser.re_key.match(line, re.IGNORECASE) or match_nn or line==' /':
            result += '\n'
            if match_nn:
                result += '\n'
                #output += "%3d: "%(i)
        elif match_array:
            # remove leading spaces from array values
            line = re.sub(r'^\s*(?=.*=)', r'', line)

        result += line
        return result

    def parse(self):
        """ main method to parse entire file """
        self.output = ''
        lastline = ''
        for i, line in enumerate(self.it):

            line = self.strip_whitespace(line)
            line = self.filter_line(line)
            line = self.parse_line(line, lastline)

            if line is not None and line != ",":
                self.output += line
                lastline = line
        return self

    def print_csv(self):
        """ print to output as CSV """
        re_key_val = re.compile(r"(\w[^ ]*)\s*=\s*(.*)$")

        print_output = ''
        it = iter(self.output.splitlines())

        for i, line in enumerate(it):
            # Remove ending comma, I know it's in the original, but it's ugly
            line = re.sub(',$', '', line)
            # Normal key/values
            if re_key_val.match(line):
                print_output += re_key_val.sub(r'"\1","\2"', line, re.IGNORECASE) + '\n'
            else:
                print_output += '"%s",""\n' % (line)
        print print_output

def main():
    """ sets up variables for pipe and then prints the parse as CSV """
    tmp_settings_file = '/tmp/TMP.gem_settings.nml'
    script_path = os.path.dirname(os.path.realpath(__file__))

    # Get namelist
    nlfile = ''
    if len(sys.argv) >= 2:
        nlfile = sys.argv[1]

    # First, remove some of the known replacement strings
    # Often the 'aftermod' file should be used (which doesn't include these)
    # rather than the 'beforemod' fileo
    #
    # In any case, skip this for now.
    if (False):
        f = open(nlfile,'r')
        output = ''
        for line in f.readlines():
            # Start and end dates
            line = re.sub(r"((strt_|end_).*)\s*=\s*('|\")[a-zA-Z].*('|\")\s*,", r"\1 = '0',", line)
            # Other known constants
            line = re.sub(r"=(\s*)PTOPO_NPE.", r"=\1 0 ", line)
            output = output + line
            f = open(tmp_settings_file,'w')
            f.write(output)
            f.close()
            nlfile = tmp_settings_file

    proc = Popen("%s/_readNamelist %s" \
            %(script_path, nlfile), stdout=PIPE, stderr=None, shell=True)

    my_parse = NSParser(proc)
    #print my_parse.parse().output
    my_parse.parse().print_csv()

    if os.path.isfile(tmp_settings_file):
        os.unlink(tmp_settings_file)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import print_function

import multiprocessing
import time

class SafeValue(object):
    """ Thread safe value. Uses locks.

    Source: http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing
    """
    def __init__(self, initval=0):
        self.val = multiprocessing.Value('i', initval)
        self.lock = multiprocessing.Lock()

    def increment(self):
        with self.lock:
            self.val.value += 1

    def decrement(self):
        with self.lock:
            self.val.value -= 1

    def set(self, n):
        with self.lock:
            self.val.value = n

    def value(self):
        with self.lock:
            return self.val.value

class QueuedPool(object):
    """ Provides capability of creating a pool of workers which will process jobs in a given queue, and the
        input queue can be updated in another thread.

        The workers will process the queue until the pool is deliberately closed. All results are stored in an
        output queue. It is also possible to change the number of workers in a pool during runtime.

    Arguments:
        func: [function] Worker function to which the arguments from the queue will be passed

    Keyword arguments:
        cores: [int] Number of CPU cores to use. None by default.

    """
    def __init__(self, func, cores=None):

        # If the cores are not given, use all available cores
        if cores is None:
            cores = multiprocessing.cpu_count()

        if cores is None:
            cores = 1

        self.cores = SafeValue(cores)

        # Initialize queues
        self.input_queue = multiprocessing.Queue()
        self.output_queue = multiprocessing.Queue()

        self.func = func
        self.pool = None

        self.total_jobs = SafeValue()
        self.active_workers = SafeValue()
        self.kill_workers = multiprocessing.Event()

        # Start the pool with the given parameters - this will wait until the input queue is given jobs
        self.startPool()

    def _workerFunc(self, func):
        """ A wrapper function for the given worker function. Handles the queue operations. """

        self.active_workers.increment()

        while True:

            # Get the function arguments (block until available)
            args = self.input_queue.get(True)

            # The 'poison pill' for killing the worker when closing is requested
            if args is None:
                break

            # Call the original worker function and collect results
            result = func(*args)

            # Save the results to an output queue
            self.output_queue.put(result)

            # Exit if exit is requested
            if self.kill_workers.is_set():
                break

        self.active_workers.decrement()

    def startPool(self):
        """ Start the pool with the given worker function and number of cores. """

        # Initialize the pool of workers with the given number of worker cores
        # Comma in the argument list is a must!
        self.pool = multiprocessing.Pool(self.cores.value(), self._workerFunc, (self.func, ))

    def closePool(self):
        """ Wait until all jobs are done and close the pool. """

        if self.pool is not None:

            # Wait until the input queue is empty, then close the pool
            while True:

                # If all jobs are done, close the pool
                if self.output_queue.qsize() == self.total_jobs.value():

                    # Insert the 'poison pill' to the queue, to kill all workers
                    for i in range(self.cores.value()):
                        self.input_queue.put(None)

                    # Wait until the pills are 'swallowed'
                    while self.input_queue.qsize():
                        time.sleep(0.1)

                    # Close the pool and wait for all threads to terminate
                    self.pool.close()
                    self.pool.terminate()
                    self.pool.join()

                    return

                else:
                    time.sleep(0.01)

    def updateCoreNumber(self, cores=None):
        """ Update the number of cores/workers used by the pool.

        Arguments:
            cores: [int] Number of CPU cores to use. None by default.

        """

        # Kill the workers
        self.kill_workers.set()

        # Wait until all workers have exited
        while self.active_workers.value() > 0:
            time.sleep(0.1)

        # Join the previous pool
        self.pool.close()
        self.pool.terminate()
        self.pool.join()

        self.kill_workers.clear()

        # If cores were not given, use all available cores
        if cores is None:
            cores = multiprocessing.cpu_count()

        self.cores.set(cores)

        # Init a new pool
        self.startPool()

    def addJob(self, job):
        """ Add a job to the input queue. Job can be a list of arguments for the worker function. If a list is
            not given, the arguments will be wrapped in the list.

        """

        # Track the total number of jobs received
        self.total_jobs.increment()

        if not isinstance(job, list):
            job = [job]

        self.input_queue.put(job)

    def getResults(self):
        """ Get the results from the output queue and store them in a list. The output list will be returned.
        """

        results = []

        # Get all elements in the output queue
        if not self.output_queue.empty():
            while True:
                try:
                    results.append(self.output_queue.get(False))
                except:
                    break

        return results

def exampleWorker(in_str, in_num):
    """ An example worker function. """

    print('Got:', in_str, in_num)

    t1 = time.time()
    while True:
        if time.time() - t1 > 10:
            break

    return in_str + " " + str(in_num) + " X"

if __name__ == "__main__":

    # Initialize the pool with only one core
    workpool = QueuedPool(exampleWorker, cores=1)

    # Give the pool something to do
    for i in range(2):

        workpool.addJob(["hello", i])

        time.sleep(0.1)

        workpool.addJob(["world", i])

    time.sleep(2)

    print('Updating cores...')

    # Use all available cores
    workpool.updateCoreNumber(3)

    print('Adding more jobs...')

    # Give the pool some more work to do
    for i in range(4):
        workpool.addJob(["test1", i])
        time.sleep(0.05)
        workpool.addJob(["test2", i])

    print('Closing the pool...')

    # Wait for everything to finish and close the pool
    workpool.closePool()

    # Print out the results
    results = workpool.getResults()

    print(results)<commit_msg>fixed result fetching hanging<commit_after>from __future__ import print_function

import multiprocessing
import time

class SafeValue(object):
    """ Thread safe value. Uses locks.

    Source: http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing
    """
    def __init__(self, initval=0):
        self.val = multiprocessing.Value('i', initval)
        self.lock = multiprocessing.Lock()

    def increment(self):
        with self.lock:
            self.val.value += 1

    def decrement(self):
        with self.lock:
            self.val.value -= 1

    def set(self, n):
        with self.lock:
            self.val.value = n

    def value(self):
        with self.lock:
            return self.val.value

class QueuedPool(object):
    """ Provides capability of creating a pool of workers which will process jobs in a given queue, and the
        input queue can be updated in another thread.

        The workers will process the queue until the pool is deliberately closed. All results are stored in an
        output queue. It is also possible to change the number of workers in a pool during runtime.

    Arguments:
        func: [function] Worker function to which the arguments from the queue will be passed

    Keyword arguments:
        cores: [int] Number of CPU cores to use. None by default.

    """
    def __init__(self, func, cores=None):

        # If the cores are not given, use all available cores
        if cores is None:
            cores = multiprocessing.cpu_count()

        if cores is None:
            cores = 1

        self.cores = SafeValue(cores)

        # Initialize queues (for some reason queues from Manager need to be created, otherwise they are
        # blocking when using get_nowait)
        self.input_queue = multiprocessing.Manager().Queue()
        self.output_queue = multiprocessing.Manager().Queue()

        self.func = func
        self.pool = None

        self.total_jobs = SafeValue()
        self.active_workers = SafeValue()
        self.kill_workers = multiprocessing.Event()

        # Start the pool with the given parameters - this will wait until the input queue is given jobs
        self.startPool()

    def _workerFunc(self, func):
        """ A wrapper function for the given worker function. Handles the queue operations. """

        self.active_workers.increment()

        while True:

            # Get the function arguments (block until available)
            args = self.input_queue.get(True)

            # The 'poison pill' for killing the worker when closing is requested
            if args is None:
                break

            # Call the original worker function and collect results
            result = func(*args)

            # Save the results to an output queue
            self.output_queue.put(result)

            # Exit if exit is requested
            if self.kill_workers.is_set():
                break

        self.active_workers.decrement()

    def startPool(self):
        """ Start the pool with the given worker function and number of cores. """

        # Initialize the pool of workers with the given number of worker cores
        # Comma in the argument list is a must!
        self.pool = multiprocessing.Pool(self.cores.value(), self._workerFunc, (self.func, ))

    def closePool(self):
        """ Wait until all jobs are done and close the pool. """

        if self.pool is not None:

            # Wait until the input queue is empty, then close the pool
            while True:

                # If all jobs are done, close the pool
                if self.output_queue.qsize() == self.total_jobs.value():

                    # Insert the 'poison pill' to the queue, to kill all workers
                    for i in range(self.cores.value()):
                        self.input_queue.put(None)

                    # Wait until the pills are 'swallowed'
                    while self.input_queue.qsize():
                        time.sleep(0.1)

                    # Close the pool and wait for all threads to terminate
                    self.pool.close()
                    self.pool.terminate()
                    self.pool.join()

                    break

                else:
                    time.sleep(0.01)

    def updateCoreNumber(self, cores=None):
        """ Update the number of cores/workers used by the pool.

        Arguments:
            cores: [int] Number of CPU cores to use. None by default.

        """

        # Kill the workers
        self.kill_workers.set()

        # Wait until all workers have exited
        while self.active_workers.value() > 0:
            time.sleep(0.1)

        # Join the previous pool
        self.pool.close()
        self.pool.terminate()
        self.pool.join()

        self.kill_workers.clear()

        # If cores were not given, use all available cores
        if cores is None:
            cores = multiprocessing.cpu_count()

        self.cores.set(cores)

        # Init a new pool
        self.startPool()

    def addJob(self, job):
        """ Add a job to the input queue. Job can be a list of arguments for the worker function. If a list is
            not given, the arguments will be wrapped in the list.

        """

        # Track the total number of jobs received
        self.total_jobs.increment()

        if not isinstance(job, list):
            job = [job]

        self.input_queue.put(job)

    def allDone(self):
        """ If all jobs are done, return True.
        """

        if self.output_queue.qsize() == self.total_jobs.value():
            return True

        else:
            return False

    def getResults(self):
        """ Get the results from the output queue and store them in a list. The output list will be returned.
        """

        results = []

        # Get all elements in the output queue
        if not self.output_queue.empty():
            while True:
                try:
                    results.append(self.output_queue.get_nowait())
                except:
                    break

        return results

def exampleWorker(in_str, in_num):
    """ An example worker function. """

    print('Got:', in_str, in_num)

    t1 = time.time()
    while True:
        if time.time() - t1 > 10:
            break

    return in_str + " " + str(in_num) + " X"

if __name__ == "__main__":

    # Initialize the pool with only one core
    workpool = QueuedPool(exampleWorker, cores=1)

    # Give the pool something to do
    for i in range(2):

        workpool.addJob(["hello", i])

        time.sleep(0.1)

        workpool.addJob(["world", i])

    time.sleep(2)

    print('Updating cores...')

    # Use all available cores
    workpool.updateCoreNumber(3)

    print('Adding more jobs...')

    # Give the pool some more work to do
    for i in range(4):
        workpool.addJob(["test1", i])
        time.sleep(0.05)
        workpool.addJob(["test2", i])

    print('Closing the pool...')

    # Wait for everything to finish and close the pool
    workpool.closePool()

    # Print out the results
    results = workpool.getResults()

    print(results)<|endoftext|>


<commit_before>"""This class is responsible for making and managing a board canvas that
will draw an interactive board and allow players to move elements onto the
board. """

import tkinter
import Board
import Location
import Market
from GameConstants import *

"""Size for the wall locations"""
WALL_WIDTH = 30
"""Size of each grid square in pixels"""
GRID_SIZE = 40
"""Size of the gap between grid locations"""
GRID_GAP = 10

#Load different images
BUILDING_IMAGES = {}
WALL_IMAGES = []

class BoardCanvas(tkinter.Tk):
    def __init__(self, board):
        tkinter.Tk.__init__(self)
        self.can = tkinter.Canvas(width=(GRID_SIZE * 3 + GRID_SIZE * Board.get_columns(board) + GRID_GAP * (Board.get_columns(board) + 2)),
            height=(GRID_SIZE * 3 + GRID_SIZE * Board.get_rows(board) + GRID_GAP * (Board.get_rows(board) + 2)))

        self.drag_data = drag_data = {"x": 0, "y": 0, "item": None}
        self.board = board
        self.can.pack()

        self.tower_image = tkinter.PhotoImage(file="Assets/Tower.gif")
        self.tower_image = self.tower_image.subsample(self.tower_image.width() // GRID_SIZE, self.tower_image.height() // GRID_SIZE)
        self.well_image = tkinter.PhotoImage(file = "Assets/Well.gif")
        self.well_image = self.well_image.subsample(self.well_image.width() // GRID_SIZE, self.well_image.height() // GRID_SIZE)
        self.merchant_image = None

        if BUILDING_IMAGES == {}:
            for color in BUILDINGS_COLORS:
                img = tkinter.PhotoImage(file="Assets/Building_" + color[0] + ".gif")
                BUILDING_IMAGES[color] = img.subsample(img.width() // GRID_SIZE, img.height() // GRID_SIZE)
            img = tkinter.PhotoImage(file="Assets/Wall_North.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (GRID_SIZE + GRID_GAP + 2), img.height() // (WALL_WIDTH)))
            img = tkinter.PhotoImage(file="Assets/Wall_South.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (GRID_SIZE + GRID_GAP + 2), img.height() // (WALL_WIDTH)))

            img = tkinter.PhotoImage(file="Assets/Wall_East.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (WALL_WIDTH), img.height() // (GRID_SIZE + GRID_GAP + 2)))
            img = tkinter.PhotoImage(file="Assets/Wall_West.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (WALL_WIDTH), img.height() // (GRID_SIZE + GRID_GAP + 2)))

        rows = Board.get_rows(self.board)
        columns = Board.get_columns(self.board)

        max_x = GRID_GAP * (columns) + GRID_SIZE * (columns + 1)
        max_y = GRID_GAP * (rows) + GRID_SIZE * (rows + 1)

        self.offx, self.offy = GRID_SIZE + GRID_GAP * 2, GRID_SIZE * 2 + GRID_GAP

        self.columns = Board.get_columns(board)
        self.rows = Board.get_rows(board)

        self.draw_grid_lines()

        self.can.tag_bind("token", "<ButtonPress-1>", self.OnTokenButtonPress)
        self.can.tag_bind("token", "<ButtonRelease-1>", self.OnTokenButtonRelease)
        self.can.tag_bind("token", "<B1-Motion>", self.OnTokenMotion)

    def setup(self):
        board = self.board
        if self.merchant_image == None:
            self.merchant_image = tkinter.PhotoImage(file = "Assets/Merchant.gif")
            self.merchant_image = self.merchant_image.subsample(self.merchant_image.width() // GRID_SIZE, self.merchant_image.height() // GRID_SIZE)

        board = self.board
        self.elements = [[None] * Board.get_columns(board)] * Board.get_rows(board);

        self.towers = []
        for i in range(1, 5):
            self.can.create_image(self.get_tower_pixels(i), image = self.tower_image)
        well = Board.get_well(self.board)
        self.well = self.can.create_image(self.get_board_pixels(well), image = self.well_image)
        self.add_merchant_to_grid(Market.get_active_market_street(Board.get_market(self.board))[0])

    def add_merchant_to_grid(self, location):
        """Adds a merchant to a specified location on the grid"""
        print(location)
        return self.can.create_image(self.get_board_pixels(location), image= self.merchant_image)

    def add_wall_to_grid(self, side, index):
        """Adds a static wall to the grid. Wall is the wall, wall 'N' is
        the north wall, wall 'W' is the west wall, wall 'E' is east wall, wall
        'S' is the south wall. index is the index of the wall, 0 is either top
        or leftmost wall."""
        if side == 'N':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[0])
        elif side == 'S':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[1])
        elif side == 'E':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[3])
        elif side == 'W':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[2])

    def get_board_pixels(self, location):
        """Gets the center of an grid square for a specific Location for the
        board returned as (x, y)"""
        col = Location.get_column(location)
        row = Location.get_row(location)
        return self.offx + col * (GRID_SIZE + 2) + (col + 1) * GRID_GAP, \
                self.offy + row * (GRID_SIZE  + 2) + (row + 1) * GRID_GAP

    def add_building_to_grid(self, color, location):
        """Adds a bulding graphic to the board at a specified location"""
        return self.can.create_image(self.get_board_pixels(location), image=BUILDING_IMAGES[color])

    def get_wall_pixels(self, wall, index):
        """Gets the cetner of a wall location. Wall is the wall, wall 'N' is
        the north wall, wall 'W' is the west wall, wall 'E' is east wall, wall
        'S' is the south wall. index is the index of the wall, 0 is either top
        or leftmost wall."""
        if wall == 'N':
            x, y = self.get_board_pixels(Location.make_location(-1, index));
            return x, y
        if wall == 'E':
            x, y = self.get_board_pixels(Location.make_location(index, -1));
            return x, y
        if wall == 'S':
            x, y = self.get_board_pixels(Location.make_location(self.rows, index));
            return x, y
        if wall == 'W':
            x, y = self.get_board_pixels(Location.make_location(index, self.columns));
            return x, y

    def get_tower_pixels(self, tower_number):
        """Gets the center of a tower given the tower number"""
        if tower_number == 1:
            x, y = self.get_board_pixels(Location.make_location(-1,-1))
            return x + (GRID_GAP) // 2, y + (GRID_GAP) // 2
        if tower_number == 2:
            x, y = self.get_board_pixels(Location.make_location(-1,self.columns))
            return x - (GRID_GAP) // 2, y + (GRID_GAP) // 2
        if tower_number == 3:
            x, y = self.get_board_pixels(Location.make_location(self.rows,-1))
            return x + (GRID_GAP) // 2, y - (GRID_GAP) // 2
        if tower_number == 4:
            x, y = self.get_board_pixels(Location.make_location(self.rows,self.columns))
            return x - (GRID_GAP) // 2, y - (GRID_GAP) // 2

    def draw_grid_lines(self):
        """Draws lines for the different grid boxes on the screen"""
        for tower in range(1, 5):
            #Draw box for towers
            x, y = self.get_tower_pixels(tower)
            self.can.create_line(x - GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y - GRID_SIZE // 2)
            self.can.create_line(x - GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x - GRID_SIZE // 2, y + GRID_SIZE // 2)
            self.can.create_line(x + GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y + GRID_SIZE // 2)
            self.can.create_line(x - GRID_SIZE // 2, y + GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y + GRID_SIZE // 2)

        x1,y1 = self.get_board_pixels(Location.make_location(0,0))
        x2,y2 = self.get_board_pixels(Location.make_location(self.rows, self.columns))
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2, \
                y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                x2 - (GRID_SIZE + GRID_GAP) // 2, \
                y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH)
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2, \
                y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                x2 - (GRID_SIZE + GRID_GAP) // 2, \
                y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH)
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                y1 - (GRID_SIZE + GRID_GAP) // 2, \
                x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                y2 - (GRID_SIZE + GRID_GAP) // 2)
        self.can.create_line(x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                y1 - (GRID_SIZE + GRID_GAP) // 2, \
                x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                y2 - (GRID_SIZE + GRID_GAP) // 2)

        for row in range(0, self.rows + 1):
            x1,y1 = self.get_board_pixels(Location.make_location(row, 0))
            x2,y2 = self.get_board_pixels(Location.make_location(row, self.columns))
            self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH,y1 - (GRID_SIZE + GRID_GAP) // 2, \
                    x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH,y2 - (GRID_SIZE + GRID_GAP) // 2)
        for column in range(0, self.columns + 1):
            x1,y1 = self.get_board_pixels(Location.make_location(0, column))
            x2,y2 = self.get_board_pixels(Location.make_location(self.rows, column))
            self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2,y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                    x2 - (GRID_SIZE + GRID_GAP) // 2,y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH)

    def add_moveable_building(self, color, coords):
        """Adds a moveable building of a specified color at an x and y"""
        return self.can.create_image(coords, image=BUILDING_IMAGES[color], tags="token")

    def OnTokenButtonPress(self, event):
        '''Being drag of an object'''
        # record the item and its location
        can = self.can
        self.drag_data["item"] = can.find_closest(event.x, event.y)[0]
        self.drag_data["x"] = event.x
        self.drag_data["y"] = event.y

    def OnTokenButtonRelease(self, event):
        '''End drag of an object'''
        # reset the drag information
        can = self.can
        self.drag_data["item"] = None
        self.drag_data["x"] = 0
        self.drag_data["y"] = 0

    def OnTokenMotion(self, event):
        '''Handle dragging of an object'''
        # compute how much this object has moved
        can = self.can
        delta_x = event.x - self.drag_data["x"]
        delta_y = event.y - self.drag_data["y"]
        # move the object the appropriate amount
        can.move(self.drag_data["item"], delta_x, delta_y)
        # record the new position
        self.drag_data["x"] = event.x
        self.drag_data["y"] = event.y

if __name__ == "__main__":
    board_canvas = BoardCanvas(Board.make_board(11,16))
    board_canvas.setup()
    built = []
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(0,0)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(0,1)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(1,0)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[3], Location.make_location(10,15)))
    built.append(board_canvas.add_wall_to_grid('S', 0))
    built.append(board_canvas.add_wall_to_grid('S', 1))
    built.append(board_canvas.add_wall_to_grid('S', 2))
    built.append(board_canvas.add_wall_to_grid('S', 3))
    built.append(board_canvas.add_wall_to_grid('W', 9))
    built.append(board_canvas.add_wall_to_grid('W', 10))
    built.append(board_canvas.add_wall_to_grid('N', 15))
    built.append(board_canvas.add_wall_to_grid('E', 10))
    print(built)
    board_canvas.mainloop()
<commit_msg>fixed get_wall_pixels function<commit_after>"""This class is responsible for making and managing a board canvas that
will draw an interactive board and allow players to move elements onto the
board. """

import tkinter
import Board
import Location
import Market
from GameConstants import *

"""Size for the wall locations"""
WALL_WIDTH = 30
"""Size of each grid square in pixels"""
GRID_SIZE = 40
"""Size of the gap between grid locations"""
GRID_GAP = 10

#Load different images
BUILDING_IMAGES = {}
WALL_IMAGES = []

class BoardCanvas(tkinter.Tk):
    def __init__(self, board):
        tkinter.Tk.__init__(self)
        self.can = tkinter.Canvas(width=(GRID_SIZE * 3 + GRID_SIZE * Board.get_columns(board) + GRID_GAP * (Board.get_columns(board) + 2)),
            height=(GRID_SIZE * 3 + GRID_SIZE * Board.get_rows(board) + GRID_GAP * (Board.get_rows(board) + 2)))

        self.drag_data = drag_data = {"x": 0, "y": 0, "item": None}
        self.board = board
        self.can.pack()

        self.tower_image = tkinter.PhotoImage(file="Assets/Tower.gif")
        self.tower_image = self.tower_image.subsample(self.tower_image.width() // GRID_SIZE, self.tower_image.height() // GRID_SIZE)
        self.well_image = tkinter.PhotoImage(file = "Assets/Well.gif")
        self.well_image = self.well_image.subsample(self.well_image.width() // GRID_SIZE, self.well_image.height() // GRID_SIZE)
        self.merchant_image = None

        if BUILDING_IMAGES == {}:
            for color in BUILDINGS_COLORS:
                img = tkinter.PhotoImage(file="Assets/Building_" + color[0] + ".gif")
                BUILDING_IMAGES[color] = img.subsample(img.width() // GRID_SIZE, img.height() // GRID_SIZE)
            img = tkinter.PhotoImage(file="Assets/Wall_North.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (GRID_SIZE + GRID_GAP + 2), img.height() // (WALL_WIDTH)))
            img = tkinter.PhotoImage(file="Assets/Wall_South.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (GRID_SIZE + GRID_GAP + 2), img.height() // (WALL_WIDTH)))

            img = tkinter.PhotoImage(file="Assets/Wall_East.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (WALL_WIDTH), img.height() // (GRID_SIZE + GRID_GAP + 2)))
            img = tkinter.PhotoImage(file="Assets/Wall_West.gif")
            WALL_IMAGES.append(img.subsample(img.width() // (WALL_WIDTH), img.height() // (GRID_SIZE + GRID_GAP + 2)))

        rows = Board.get_rows(self.board)
        columns = Board.get_columns(self.board)

        max_x = GRID_GAP * (columns) + GRID_SIZE * (columns + 1)
        max_y = GRID_GAP * (rows) + GRID_SIZE * (rows + 1)

        self.offx, self.offy = GRID_SIZE + GRID_GAP * 2, GRID_SIZE * 2 + GRID_GAP

        self.columns = Board.get_columns(board)
        self.rows = Board.get_rows(board)

        self.draw_grid_lines()

        self.can.tag_bind("token", "<ButtonPress-1>", self.OnTokenButtonPress)
        self.can.tag_bind("token", "<ButtonRelease-1>", self.OnTokenButtonRelease)
        self.can.tag_bind("token", "<B1-Motion>", self.OnTokenMotion)

    def setup(self):
        board = self.board
        if self.merchant_image == None:
            self.merchant_image = tkinter.PhotoImage(file = "Assets/Merchant.gif")
            self.merchant_image = self.merchant_image.subsample(self.merchant_image.width() // GRID_SIZE, self.merchant_image.height() // GRID_SIZE)

        board = self.board
        self.elements = [[None] * Board.get_columns(board)] * Board.get_rows(board);

        self.towers = []
        for i in range(1, 5):
            self.can.create_image(self.get_tower_pixels(i), image = self.tower_image)
        well = Board.get_well(self.board)
        self.well = self.can.create_image(self.get_board_pixels(well), image = self.well_image)
        self.add_merchant_to_grid(Market.get_active_market_street(Board.get_market(self.board))[0])

    def add_merchant_to_grid(self, location):
        """Adds a merchant to a specified location on the grid"""
        print(location)
        return self.can.create_image(self.get_board_pixels(location), image= self.merchant_image)

    def add_wall_to_grid(self, side, index):
        """Adds a static wall to the grid. Wall is the wall, wall 'N' is
        the north wall, wall 'W' is the west wall, wall 'E' is east wall, wall
        'S' is the south wall. index is the index of the wall, 0 is either top
        or leftmost wall."""
        if side == 'N':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[0])
        elif side == 'S':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[1])
        elif side == 'E':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[2])
        elif side == 'W':
            return self.can.create_image(self.get_wall_pixels(side, index), image=WALL_IMAGES[3])

    def get_board_pixels(self, location):
        """Gets the center of an grid square for a specific Location for the
        board returned as (x, y)"""
        col = Location.get_column(location)
        row = Location.get_row(location)
        return self.offx + col * (GRID_SIZE + 2) + (col + 1) * GRID_GAP, \
                self.offy + row * (GRID_SIZE  + 2) + (row + 1) * GRID_GAP

    def add_building_to_grid(self, color, location):
        """Adds a bulding graphic to the board at a specified location"""
        return self.can.create_image(self.get_board_pixels(location), image=BUILDING_IMAGES[color])

    def get_wall_pixels(self, wall, index):
        """Gets the center of a wall location. Wall is the wall, wall 'N' is
        the north wall, wall 'W' is the west wall, wall 'E' is east wall, wall
        'S' is the south wall. index is the index of the wall, 0 is either top
        or leftmost wall."""
        if wall == 'N':
            x, y = self.get_board_pixels(Location.make_location(-1, index));
            return x, y
        if wall == 'E':
            x, y = self.get_board_pixels(Location.make_location(index, self.columns));
            return x, y
        if wall == 'S':
            x, y = self.get_board_pixels(Location.make_location(self.rows, index));
            return x, y
        if wall == 'W':
            x, y = self.get_board_pixels(Location.make_location(index, -1));
            return x, y

    def get_tower_pixels(self, tower_number):
        """Gets the center of a tower given the tower number"""
        if tower_number == 1:
            x, y = self.get_board_pixels(Location.make_location(-1,-1))
            return x + (GRID_GAP) // 2, y + (GRID_GAP) // 2
        if tower_number == 2:
            x, y = self.get_board_pixels(Location.make_location(-1,self.columns))
            return x - (GRID_GAP) // 2, y + (GRID_GAP) // 2
        if tower_number == 3:
            x, y = self.get_board_pixels(Location.make_location(self.rows,-1))
            return x + (GRID_GAP) // 2, y - (GRID_GAP) // 2
        if tower_number == 4:
            x, y = self.get_board_pixels(Location.make_location(self.rows,self.columns))
            return x - (GRID_GAP) // 2, y - (GRID_GAP) // 2

    def draw_grid_lines(self):
        """Draws lines for the different grid boxes on the screen"""
        for tower in range(1, 5):
            #Draw box for towers
            x, y = self.get_tower_pixels(tower)
            self.can.create_line(x - GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y - GRID_SIZE // 2)
            self.can.create_line(x - GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x - GRID_SIZE // 2, y + GRID_SIZE // 2)
            self.can.create_line(x + GRID_SIZE // 2, y - GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y + GRID_SIZE // 2)
            self.can.create_line(x - GRID_SIZE // 2, y + GRID_SIZE // 2,
                    x + GRID_SIZE // 2, y + GRID_SIZE // 2)

        x1,y1 = self.get_board_pixels(Location.make_location(0,0))
        x2,y2 = self.get_board_pixels(Location.make_location(self.rows, self.columns))
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2, \
                y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                x2 - (GRID_SIZE + GRID_GAP) // 2, \
                y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH)
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2, \
                y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                x2 - (GRID_SIZE + GRID_GAP) // 2, \
                y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH)
        self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                y1 - (GRID_SIZE + GRID_GAP) // 2, \
                x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                y2 - (GRID_SIZE + GRID_GAP) // 2)
        self.can.create_line(x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                y1 - (GRID_SIZE + GRID_GAP) // 2, \
                x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH, \
                y2 - (GRID_SIZE + GRID_GAP) // 2)

        for row in range(0, self.rows + 1):
            x1,y1 = self.get_board_pixels(Location.make_location(row, 0))
            x2,y2 = self.get_board_pixels(Location.make_location(row, self.columns))
            self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH,y1 - (GRID_SIZE + GRID_GAP) // 2, \
                    x2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH,y2 - (GRID_SIZE + GRID_GAP) // 2)
        for column in range(0, self.columns + 1):
            x1,y1 = self.get_board_pixels(Location.make_location(0, column))
            x2,y2 = self.get_board_pixels(Location.make_location(self.rows, column))
            self.can.create_line(x1 - (GRID_SIZE + GRID_GAP) // 2,y1 - (GRID_SIZE + GRID_GAP) // 2 - WALL_WIDTH, \
                    x2 - (GRID_SIZE + GRID_GAP) // 2,y2 - (GRID_SIZE + GRID_GAP) // 2 + WALL_WIDTH)

    def add_moveable_building(self, color, coords):
        """Adds a moveable building of a specified color at an x and y"""
        return self.can.create_image(coords, image=BUILDING_IMAGES[color], tags="token")

    def OnTokenButtonPress(self, event):
        '''Being drag of an object'''
        # record the item and its location
        can = self.can
        self.drag_data["item"] = can.find_closest(event.x, event.y)[0]
        self.drag_data["x"] = event.x
        self.drag_data["y"] = event.y

    def OnTokenButtonRelease(self, event):
        '''End drag of an object'''
        # reset the drag information
        can = self.can
        self.drag_data["item"] = None
        self.drag_data["x"] = 0
        self.drag_data["y"] = 0

    def OnTokenMotion(self, event):
        '''Handle dragging of an object'''
        # compute how much this object has moved
        can = self.can
        delta_x = event.x - self.drag_data["x"]
        delta_y = event.y - self.drag_data["y"]
        # move the object the appropriate amount
        can.move(self.drag_data["item"], delta_x, delta_y)
        # record the new position
        self.drag_data["x"] = event.x
        self.drag_data["y"] = event.y

if __name__ == "__main__":
    board_canvas = BoardCanvas(Board.make_board(11,16))
    board_canvas.setup()
    built = []
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(0,0)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(0,1)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[0], Location.make_location(1,0)))
    built.append(board_canvas.add_building_to_grid(BUILDINGS_COLORS[3], Location.make_location(10,15)))
    built.append(board_canvas.add_wall_to_grid('S', 0))
    built.append(board_canvas.add_wall_to_grid('S', 1))
    built.append(board_canvas.add_wall_to_grid('S', 2))
    built.append(board_canvas.add_wall_to_grid('S', 3))
    built.append(board_canvas.add_wall_to_grid('W', 9))
    built.append(board_canvas.add_wall_to_grid('W', 10))
    built.append(board_canvas.add_wall_to_grid('N', 15))
    built.append(board_canvas.add_wall_to_grid('E', 10))
    print(built)
    board_canvas.mainloop()
<|endoftext|>


<commit_before>"""
========================================================================
Model fitting on imbalanced dataset and comparison of methods to improve
its performance
========================================================================

This example illustrates the problem induced by learning on datasets having
imbalanced classes. Subsequently, we compare different approaches alleviating
these negative effects.

"""

# Authors: Guillaume Lemaitre <g.lemaitre58@gmail.com>
# License: MIT

print(__doc__)

###############################################################################
# Problem definition
###############################################################################

from sklearn.datasets import fetch_openml

df, y = fetch_openml('adult', version=2, as_frame=True, return_X_y=True)
# we are dropping the following features:
# - "fnlwgt": this feature was created while studying the "adult" dataset.
#   Thus, we will not use this feature which is not acquired during the survey.
# - "education-num": it is encoding the same information than "education".
#   Thus, we are removing one of these 2 features.
df = df.drop(columns=['fnlwgt', 'education-num'])

###############################################################################
# The "adult" dataset as a class ratio of about 3:1

classes_count = y.value_counts()
classes_count

###############################################################################
# This dataset is only slightly imbalanced. To better highlight the effect of
# learning from an imbalanced dataset, we will increase its ratio to 30:1

from imblearn.datasets import make_imbalance

ratio = 30
df_res, y_res = make_imbalance(
    df, y, sampling_strategy={
        classes_count.idxmin(): classes_count.max() // ratio
    }
)
y_res.value_counts()

###############################################################################
# For the rest of the notebook, we will make a single split to get training
# and testing data. Note that you should use cross-validation to have an
# estimate of the performance variation in practice.

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    df_res, y_res, stratify=y_res, random_state=42
)

###############################################################################
# As a baseline, we could use a classifier which will always predict the
# majority class independently of the features provided.

from sklearn.dummy import DummyClassifier

dummy_clf = DummyClassifier(strategy="most_frequent")
score = dummy_clf.fit(X_train, y_train).score(X_test, y_test)
print(f"Accuracy score of a dummy classifier: {score:.3f}")

##############################################################################
# Instead of using the accuracy, we can use the balanced accuracy which will
# take into account the balancing issue.

from sklearn.metrics import balanced_accuracy_score

y_pred = dummy_clf.predict(X_test)
score = balanced_accuracy_score(y_test, y_pred)
print(f"Balanced accuracy score of a dummy classifier: {score:.3f}")

###############################################################################
# Strategies to learn from an imbalanced dataset
###############################################################################

###############################################################################
# We will first define a helper function which will train a given model
# and compute both accuracy and balanced accuracy. The results will be stored
# in a dataframe

import pandas as pd

def evaluate_classifier(clf, df_scores, clf_name=None):
    from sklearn.pipeline import Pipeline
    if clf_name is None:
        if isinstance(clf, Pipeline):
            clf_name = clf[-1].__class__.__name__
        else:
            clf_name = clf.__class__.__name__
    acc = clf.fit(X_train, y_train).score(X_test, y_test)
    y_pred = clf.predict(X_test)
    bal_acc = balanced_accuracy_score(y_test, y_pred)
    clf_score = pd.DataFrame(
        {clf_name: [acc, bal_acc]},
        index=['Accuracy', 'Balanced accuracy']
    )
    df_scores = pd.concat([df_scores, clf_score], axis=1).round(decimals=3)
    return df_scores

# Let's define an empty dataframe to store the results
df_scores = pd.DataFrame()

###############################################################################
# Dummy baseline
# ..............
#
# Before to train a real machine learning model, we can store the results
# obtained with our `DummyClassifier`.

df_scores = evaluate_classifier(dummy_clf, df_scores, "Dummy")
df_scores

###############################################################################
# Linear classifier baseline
# ..........................
#
# We will create a machine learning pipeline using a `LogisticRegression`
# classifier. In this regard, we will need to one-hot encode the categorical
# columns and standardized the numerical columns before to inject the data into
# the `LogisticRegression` classifier.
#
# First, we define our numerical and categorical pipelines.

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import make_pipeline

num_pipe = make_pipeline(
    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
)
cat_pipe = make_pipeline(
    SimpleImputer(strategy="constant", fill_value="missing"),
    OneHotEncoder(handle_unknown="ignore")
)

###############################################################################
# Then, we can create a preprocessor which will dispatch the categorical
# columns to the categorical pipeline and the numerical columns to the
# numerical pipeline

import numpy as np
from sklearn.compose import ColumnTransformer
from sklearn.compose import make_column_selector as selector

preprocessor_linear = ColumnTransformer(
    [("num-pipe", num_pipe, selector(dtype_include=np.number)),
     ("cat-pipe", cat_pipe, selector(dtype_include=pd.CategoricalDtype))],
    n_jobs=2
)

###############################################################################
# Finally, we connect our preprocessor with our `LogisticRegression`. We can
# then evaluate our model.

from sklearn.linear_model import LogisticRegression

lr_clf = make_pipeline(
    preprocessor_linear, LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(lr_clf, df_scores, "LR")
df_scores

###############################################################################
# We can see that our linear model is learning slightly better than our dummy
# baseline. However, it is impacted by the class imbalance.
#
# We can verify that something similar is happening with a tree-based model
# such as `RandomForestClassifier`. With this type of classifier, we will not
# need to scale the numerical data, and we will only need to ordinal encode the
# categorical data.

from sklearn.preprocessing import OrdinalEncoder
from sklearn.ensemble import RandomForestClassifier

cat_pipe = make_pipeline(
    SimpleImputer(strategy="constant", fill_value="missing"),
    OrdinalEncoder()
)

preprocessor_tree = ColumnTransformer(
    [("num-pipe", num_pipe, selector(dtype_include=np.number)),
     ("cat-pipe", cat_pipe, selector(dtype_include=pd.CategoricalDtype))],
    n_jobs=2
)

rf_clf = make_pipeline(
    preprocessor_tree, RandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(rf_clf, df_scores, "RF")
df_scores

###############################################################################
# The `RandomForestClassifier` is as well affected by the class imbalanced,
# slightly less than the linear model. Now, we will present different approach
# to improve the performance of these 2 models.
#
# Use `class_weight`
# ..................
#
# Most of the models in `scikit-learn` have a parameter `class_weight`. This
# parameter will affect the computation of the loss in linear model or the
# criterion in the tree-based model to penalize differently a false
# classification from the minority and majority class. We can set
# `class_weight="balanced"` such that the weight applied is inversely
# proportional to the class frequency. We test this parametrization in both
# linear model and tree-based model.

lr_clf.set_params(logisticregression__class_weight="balanced")
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with class weight"
)
df_scores

###############################################################################
#

rf_clf.set_params(randomforestclassifier__class_weight="balanced")
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with class weight"
)
df_scores

###############################################################################
# We can see that using `class_weight` was really effective for the linear
# model, alleviating the issue of learning from imbalanced classes. However,
# the `RandomForestClassifier` is still biased toward the majority class,
# mainly due to the criterion which is not suited enough to fight the class
# imbalance.
#
# Resample the training set during learning
# .........................................
#
# Another way is to resample the training set by under-sampling or
# over-sampling some of the samples. `imbalanced-learn` provides some samplers
# to do such processing.

from imblearn.pipeline import make_pipeline as make_pipeline_with_sampler
from imblearn.under_sampling import RandomUnderSampler

lr_clf = make_pipeline_with_sampler(
    preprocessor_linear,
    RandomUnderSampler(random_state=42),
    LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with under-sampling"
)
df_scores

###############################################################################
#

rf_clf = make_pipeline_with_sampler(
    preprocessor_tree,
    RandomUnderSampler(random_state=42),
    RandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with under-sampling"
)
df_scores

###############################################################################
# Applying a random under-sampler before the training of the linear model or
# random forest, allows to not focus on the majority class at the cost of
# making more mistake for samples in the majority class (i.e. decreased
# accuracy).
#
# We could apply any type of samplers and find which sampler is working best
# on the current dataset.
#
# Instead, we will present another way by using classifiers which will apply
# sampling internally.
#
# Use of `BalancedRandomForestClassifier` and `BalancedBaggingClassifier`
# .......................................................................
#
# We already showed that random under-sampling can be effective on decision
# tree. However, instead of under-sampling once the dataset, one could
# under-sample the original dataset before to take a bootstrap sample. This is
# the base of the `BalancedRandomForestClassifier` and
# `BalancedBaggingClassifier`.

from imblearn.ensemble import BalancedRandomForestClassifier

rf_clf = make_pipeline(
    preprocessor_tree,
    BalancedRandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(rf_clf, df_scores, "Balanced RF")
df_scores

###############################################################################
# The performance with the `BalancedRandomForestClassifier` is better than
# applying a single random under-sampling. We will use a gradient-boosting
# classifier within a `BalancedBaggingClassifier`.

from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier
from imblearn.ensemble import BalancedBaggingClassifier

bag_clf = make_pipeline(
    preprocessor_tree,
    BalancedBaggingClassifier(
        base_estimator=HistGradientBoostingClassifier(random_state=42),
        n_estimators=10, random_state=42, n_jobs=2
    )
)

df_scores = evaluate_classifier(
    bag_clf, df_scores, "Balanced bagging"
)
df_scores

###############################################################################
# This last approach is the most effective. The different under-sampling allows
# to bring some diversity for the different GBDT to learn and not focus on a
# portion of the majority class.
#
# We will repeat the same experiment but with a ratio of 100:1 and make a
# similar analysis.

###############################################################################
# Increase imbalanced ratio
###############################################################################

ratio = 100
df_res, y_res = make_imbalance(
    df, y, sampling_strategy={
        classes_count.idxmin(): classes_count.max() // ratio
    }
)
X_train, X_test, y_train, y_test = train_test_split(
    df_res, y_res, stratify=y_res, random_state=42
)

df_scores = pd.DataFrame()
df_scores = evaluate_classifier(dummy_clf, df_scores, "Dummy")
lr_clf = make_pipeline(
    preprocessor_linear, LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(lr_clf, df_scores, "LR")
rf_clf = make_pipeline(
    preprocessor_tree, RandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(rf_clf, df_scores, "RF")
lr_clf.set_params(logisticregression__class_weight="balanced")
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with class weight"
)
rf_clf.set_params(randomforestclassifier__class_weight="balanced")
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with class weight"
)
lr_clf = make_pipeline_with_sampler(
    preprocessor_linear,
    RandomUnderSampler(random_state=42),
    LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with under-sampling"
)
rf_clf = make_pipeline_with_sampler(
    preprocessor_tree,
    RandomUnderSampler(random_state=42),
    RandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with under-sampling"
)
rf_clf = make_pipeline(
    preprocessor_tree,
    BalancedRandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(rf_clf, df_scores)
df_scores = evaluate_classifier(
    bag_clf, df_scores, "Balanced bagging"
)
df_scores

###############################################################################
# When we analyse the results, we can draw similar conclusions than in the
# previous discussion. However, we can observe that the strategy
# `class_weight="balanced"` does not improve the performance when using a
# `RandomForestClassifier`. A resampling is indeed required. The most effective
# method remains the `BalancedBaggingClassifier` using a GBDT as a base
# learner.
<commit_msg>DOC rename too long title in example<commit_after>"""
==========================================================
Fitting model on imbalanced datasets and how to fight bias
==========================================================

This example illustrates the problem induced by learning on datasets having
imbalanced classes. Subsequently, we compare different approaches alleviating
these negative effects.

"""

# Authors: Guillaume Lemaitre <g.lemaitre58@gmail.com>
# License: MIT

print(__doc__)

###############################################################################
# Problem definition
###############################################################################

from sklearn.datasets import fetch_openml

df, y = fetch_openml('adult', version=2, as_frame=True, return_X_y=True)
# we are dropping the following features:
# - "fnlwgt": this feature was created while studying the "adult" dataset.
#   Thus, we will not use this feature which is not acquired during the survey.
# - "education-num": it is encoding the same information than "education".
#   Thus, we are removing one of these 2 features.
df = df.drop(columns=['fnlwgt', 'education-num'])

###############################################################################
# The "adult" dataset as a class ratio of about 3:1

classes_count = y.value_counts()
classes_count

###############################################################################
# This dataset is only slightly imbalanced. To better highlight the effect of
# learning from an imbalanced dataset, we will increase its ratio to 30:1

from imblearn.datasets import make_imbalance

ratio = 30
df_res, y_res = make_imbalance(
    df, y, sampling_strategy={
        classes_count.idxmin(): classes_count.max() // ratio
    }
)
y_res.value_counts()

###############################################################################
# For the rest of the notebook, we will make a single split to get training
# and testing data. Note that you should use cross-validation to have an
# estimate of the performance variation in practice.

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    df_res, y_res, stratify=y_res, random_state=42
)

###############################################################################
# As a baseline, we could use a classifier which will always predict the
# majority class independently of the features provided.

from sklearn.dummy import DummyClassifier

dummy_clf = DummyClassifier(strategy="most_frequent")
score = dummy_clf.fit(X_train, y_train).score(X_test, y_test)
print(f"Accuracy score of a dummy classifier: {score:.3f}")

##############################################################################
# Instead of using the accuracy, we can use the balanced accuracy which will
# take into account the balancing issue.

from sklearn.metrics import balanced_accuracy_score

y_pred = dummy_clf.predict(X_test)
score = balanced_accuracy_score(y_test, y_pred)
print(f"Balanced accuracy score of a dummy classifier: {score:.3f}")

###############################################################################
# Strategies to learn from an imbalanced dataset
###############################################################################

###############################################################################
# We will first define a helper function which will train a given model
# and compute both accuracy and balanced accuracy. The results will be stored
# in a dataframe

import pandas as pd

def evaluate_classifier(clf, df_scores, clf_name=None):
    from sklearn.pipeline import Pipeline
    if clf_name is None:
        if isinstance(clf, Pipeline):
            clf_name = clf[-1].__class__.__name__
        else:
            clf_name = clf.__class__.__name__
    acc = clf.fit(X_train, y_train).score(X_test, y_test)
    y_pred = clf.predict(X_test)
    bal_acc = balanced_accuracy_score(y_test, y_pred)
    clf_score = pd.DataFrame(
        {clf_name: [acc, bal_acc]},
        index=['Accuracy', 'Balanced accuracy']
    )
    df_scores = pd.concat([df_scores, clf_score], axis=1).round(decimals=3)
    return df_scores

# Let's define an empty dataframe to store the results
df_scores = pd.DataFrame()

###############################################################################
# Dummy baseline
# ..............
#
# Before to train a real machine learning model, we can store the results
# obtained with our `DummyClassifier`.

df_scores = evaluate_classifier(dummy_clf, df_scores, "Dummy")
df_scores

###############################################################################
# Linear classifier baseline
# ..........................
#
# We will create a machine learning pipeline using a `LogisticRegression`
# classifier. In this regard, we will need to one-hot encode the categorical
# columns and standardized the numerical columns before to inject the data into
# the `LogisticRegression` classifier.
#
# First, we define our numerical and categorical pipelines.

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import make_pipeline

num_pipe = make_pipeline(
    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
)
cat_pipe = make_pipeline(
    SimpleImputer(strategy="constant", fill_value="missing"),
    OneHotEncoder(handle_unknown="ignore")
)

###############################################################################
# Then, we can create a preprocessor which will dispatch the categorical
# columns to the categorical pipeline and the numerical columns to the
# numerical pipeline

import numpy as np
from sklearn.compose import ColumnTransformer
from sklearn.compose import make_column_selector as selector

preprocessor_linear = ColumnTransformer(
    [("num-pipe", num_pipe, selector(dtype_include=np.number)),
     ("cat-pipe", cat_pipe, selector(dtype_include=pd.CategoricalDtype))],
    n_jobs=2
)

###############################################################################
# Finally, we connect our preprocessor with our `LogisticRegression`. We can
# then evaluate our model.

from sklearn.linear_model import LogisticRegression

lr_clf = make_pipeline(
    preprocessor_linear, LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(lr_clf, df_scores, "LR")
df_scores

###############################################################################
# We can see that our linear model is learning slightly better than our dummy
# baseline. However, it is impacted by the class imbalance.
#
# We can verify that something similar is happening with a tree-based model
# such as `RandomForestClassifier`. With this type of classifier, we will not
# need to scale the numerical data, and we will only need to ordinal encode the
# categorical data.

from sklearn.preprocessing import OrdinalEncoder
from sklearn.ensemble import RandomForestClassifier

cat_pipe = make_pipeline(
    SimpleImputer(strategy="constant", fill_value="missing"),
    OrdinalEncoder()
)

preprocessor_tree = ColumnTransformer(
    [("num-pipe", num_pipe, selector(dtype_include=np.number)),
     ("cat-pipe", cat_pipe, selector(dtype_include=pd.CategoricalDtype))],
    n_jobs=2
)

rf_clf = make_pipeline(
    preprocessor_tree, RandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(rf_clf, df_scores, "RF")
df_scores

###############################################################################
# The `RandomForestClassifier` is as well affected by the class imbalanced,
# slightly less than the linear model. Now, we will present different approach
# to improve the performance of these 2 models.
#
# Use `class_weight`
# ..................
#
# Most of the models in `scikit-learn` have a parameter `class_weight`. This
# parameter will affect the computation of the loss in linear model or the
# criterion in the tree-based model to penalize differently a false
# classification from the minority and majority class. We can set
# `class_weight="balanced"` such that the weight applied is inversely
# proportional to the class frequency. We test this parametrization in both
# linear model and tree-based model.

lr_clf.set_params(logisticregression__class_weight="balanced")
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with class weight"
)
df_scores

###############################################################################
#

rf_clf.set_params(randomforestclassifier__class_weight="balanced")
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with class weight"
)
df_scores

###############################################################################
# We can see that using `class_weight` was really effective for the linear
# model, alleviating the issue of learning from imbalanced classes. However,
# the `RandomForestClassifier` is still biased toward the majority class,
# mainly due to the criterion which is not suited enough to fight the class
# imbalance.
#
# Resample the training set during learning
# .........................................
#
# Another way is to resample the training set by under-sampling or
# over-sampling some of the samples. `imbalanced-learn` provides some samplers
# to do such processing.

from imblearn.pipeline import make_pipeline as make_pipeline_with_sampler
from imblearn.under_sampling import RandomUnderSampler

lr_clf = make_pipeline_with_sampler(
    preprocessor_linear,
    RandomUnderSampler(random_state=42),
    LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with under-sampling"
)
df_scores

###############################################################################
#

rf_clf = make_pipeline_with_sampler(
    preprocessor_tree,
    RandomUnderSampler(random_state=42),
    RandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with under-sampling"
)
df_scores

###############################################################################
# Applying a random under-sampler before the training of the linear model or
# random forest, allows to not focus on the majority class at the cost of
# making more mistake for samples in the majority class (i.e. decreased
# accuracy).
#
# We could apply any type of samplers and find which sampler is working best
# on the current dataset.
#
# Instead, we will present another way by using classifiers which will apply
# sampling internally.
#
# Use of `BalancedRandomForestClassifier` and `BalancedBaggingClassifier`
# .......................................................................
#
# We already showed that random under-sampling can be effective on decision
# tree. However, instead of under-sampling once the dataset, one could
# under-sample the original dataset before to take a bootstrap sample. This is
# the base of the `BalancedRandomForestClassifier` and
# `BalancedBaggingClassifier`.

from imblearn.ensemble import BalancedRandomForestClassifier

rf_clf = make_pipeline(
    preprocessor_tree,
    BalancedRandomForestClassifier(random_state=42, n_jobs=2)
)

df_scores = evaluate_classifier(rf_clf, df_scores, "Balanced RF")
df_scores

###############################################################################
# The performance with the `BalancedRandomForestClassifier` is better than
# applying a single random under-sampling. We will use a gradient-boosting
# classifier within a `BalancedBaggingClassifier`.

from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier
from imblearn.ensemble import BalancedBaggingClassifier

bag_clf = make_pipeline(
    preprocessor_tree,
    BalancedBaggingClassifier(
        base_estimator=HistGradientBoostingClassifier(random_state=42),
        n_estimators=10, random_state=42, n_jobs=2
    )
)

df_scores = evaluate_classifier(
    bag_clf, df_scores, "Balanced bagging"
)
df_scores

###############################################################################
# This last approach is the most effective. The different under-sampling allows
# to bring some diversity for the different GBDT to learn and not focus on a
# portion of the majority class.
#
# We will repeat the same experiment but with a ratio of 100:1 and make a
# similar analysis.

###############################################################################
# Increase imbalanced ratio
###############################################################################

ratio = 100
df_res, y_res = make_imbalance(
    df, y, sampling_strategy={
        classes_count.idxmin(): classes_count.max() // ratio
    }
)
X_train, X_test, y_train, y_test = train_test_split(
    df_res, y_res, stratify=y_res, random_state=42
)

df_scores = pd.DataFrame()
df_scores = evaluate_classifier(dummy_clf, df_scores, "Dummy")
lr_clf = make_pipeline(
    preprocessor_linear, LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(lr_clf, df_scores, "LR")
rf_clf = make_pipeline(
    preprocessor_tree, RandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(rf_clf, df_scores, "RF")
lr_clf.set_params(logisticregression__class_weight="balanced")
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with class weight"
)
rf_clf.set_params(randomforestclassifier__class_weight="balanced")
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with class weight"
)
lr_clf = make_pipeline_with_sampler(
    preprocessor_linear,
    RandomUnderSampler(random_state=42),
    LogisticRegression(max_iter=1000)
)
df_scores = evaluate_classifier(
    lr_clf, df_scores, "LR with under-sampling"
)
rf_clf = make_pipeline_with_sampler(
    preprocessor_tree,
    RandomUnderSampler(random_state=42),
    RandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(
    rf_clf, df_scores, "RF with under-sampling"
)
rf_clf = make_pipeline(
    preprocessor_tree,
    BalancedRandomForestClassifier(random_state=42, n_jobs=2)
)
df_scores = evaluate_classifier(rf_clf, df_scores)
df_scores = evaluate_classifier(
    bag_clf, df_scores, "Balanced bagging"
)
df_scores

###############################################################################
# When we analyse the results, we can draw similar conclusions than in the
# previous discussion. However, we can observe that the strategy
# `class_weight="balanced"` does not improve the performance when using a
# `RandomForestClassifier`. A resampling is indeed required. The most effective
# method remains the `BalancedBaggingClassifier` using a GBDT as a base
# learner.
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import sys
import asyncio
from pprint import pprint

# -----------------------------------------------------------------------------

this_folder = os.path.dirname(os.path.abspath(__file__))
root_folder = os.path.dirname(os.path.dirname(this_folder))
sys.path.append(root_folder + '/python')
sys.path.append(this_folder)

# -----------------------------------------------------------------------------

import ccxt  # noqa: E402

# -----------------------------------------------------------------------------

exchange = ccxt.binanceusdm()
exchange.load_markets()
timeframe = '1h'
ohlcvs = []

async def fetchOHLCV(symbol):
    '''
    Wrapper around exchange.fetchOHLCV method
    :param str symbol: CCXT unified symbol
    :returns [float|str]: 1d array with a single ohlcv record with the market symbol appended
    '''
    try:
        ohlcv = exchange.fetchOHLCV(symbol, timeframe, None, 2)
        ohlcv[0].append(symbol)
        ohlcvs.append(ohlcv[0])
    except Exception as e:
        print(f'{symbol} failed fetchOHLCV with exception {e}')

def getPriceChangePercent(ohlcv):
    '''
    Gets the price change of a market as a percentage
    :param [float] ohlcv: A single ohlcv record with the market symbol appended
    :returns [float, str]: The price change as a percent with the symbol for the market
    '''
    open = ohlcv[1]
    close = ohlcv[4]
    symbol = ohlcv[6]
    priceIncrease = close - open
    increaseAsRatio = priceIncrease / open
    return [increaseAsRatio, symbol]

async def main():
    '''
    Gets the price change as a percent of every market matching type over the last timeframe matching timeframe and prints a sorted list.
    The most immediate candle is ignored because it is incomplete
    '''
    allSwapSymbols = [symbol for symbol in exchange.symbols if exchange.market(symbol)['swap']]
    await asyncio.gather(*[fetchOHLCV(symbol) for symbol in allSwapSymbols])
    priceChanges = [getPriceChangePercent(ohlcv) for ohlcv in ohlcvs]
    priceChanges.sort()
    pprint(priceChanges)

loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
<commit_msg>examples/py/sort-swap-markets-by-hourly-price-change.py uses ccxt.async_support<commit_after># -*- coding: utf-8 -*-

import os
import sys
import asyncio
from pprint import pprint

# -----------------------------------------------------------------------------

this_folder = os.path.dirname(os.path.abspath(__file__))
root_folder = os.path.dirname(os.path.dirname(this_folder))
sys.path.append(root_folder + '/python')
sys.path.append(this_folder)

# -----------------------------------------------------------------------------

import ccxt.async_support as ccxt  # noqa: E402

# -----------------------------------------------------------------------------

exchange = ccxt.binanceusdm()
timeframe = '1h'
ohlcvs = []

async def fetchOHLCV(symbol):
    '''
    Wrapper around exchange.fetchOHLCV method
    :param str symbol: CCXT unified symbol
    :returns [float|str]: 1d array with a single ohlcv record with the market symbol appended
    '''
    try:
        ohlcv = await exchange.fetchOHLCV(symbol, timeframe, None, 2)
        ohlcv[0].append(symbol)
        ohlcvs.append(ohlcv[0])
    except Exception as e:
        print(f'{symbol} failed fetchOHLCV with exception {e}')

def getPriceChangePercent(ohlcv):
    '''
    Gets the price change of a market as a percentage
    :param [float] ohlcv: A single ohlcv record with the market symbol appended
    :returns [float, str]: The price change as a percent with the symbol for the market
    '''
    open = ohlcv[1]
    close = ohlcv[4]
    symbol = ohlcv[6]
    priceIncrease = close - open
    increaseAsRatio = priceIncrease / open
    return [increaseAsRatio, symbol]

async def main():
    '''
    Gets the price change as a percent of every market matching type over the last timeframe matching timeframe and prints a sorted list.
    The most immediate candle is ignored because it is incomplete
    '''
    await exchange.load_markets()
    allSwapSymbols = [symbol for symbol in exchange.symbols if exchange.market(symbol)['swap']]
    await asyncio.gather(*[fetchOHLCV(symbol) for symbol in allSwapSymbols])
    await exchange.close()
    priceChanges = [getPriceChangePercent(ohlcv) for ohlcv in ohlcvs]
    priceChanges.sort()
    pprint(priceChanges)

loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
@brief Test the new transform prototype against streams and events
@author Swarbhanu Chatterjee
'''

from pyon.ion.transforma import TransformEventListener, TransformEventPublisher, TransformAlgorithm
from pyon.public import log
from pyon.ion.stream import SimpleStreamPublisher
from pyon.util.containers import DotDict
from pyon.util.file_sys import FileSystem
from pyon.util.int_test import IonIntegrationTestCase
from pyon.util.unit_test import IonUnitTestCase
from pyon.event.event import EventPublisher, EventSubscriber
from nose.plugins.attrib import attr
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient
from interface.objects import ProcessDefinition
from mock import Mock, sentinel, patch
import gevent

@attr('INT', group='dm')
class TransformPrototypeIntTest(IonIntegrationTestCase):
    def setUp(self):
        super(TransformPrototypeIntTest, self).setUp()

        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        self.rrc = ResourceRegistryServiceClient()
        self.event_publisher = EventPublisher()

    def test_event_processing(self):
        '''
        Test that events are processed by the transforms according to a provided algorithm
        '''

        #-------------------------------------------------------------------------------------
        # Create an event alert transform
        #-------------------------------------------------------------------------------------

        #-------------------------------------------------------------------------------------
        # The configuration for the Event Alert Transform... set up the event types to listen to
        #-------------------------------------------------------------------------------------
        configuration = {
                            'process':{
                                'event_type': 'ExampleDetectableEvent',
                                'max_count': 3,
                                'time_window': 5,
                            }
                        }

        #-------------------------------------------------------------------------------------
        # Create the process
        #-------------------------------------------------------------------------------------
        pid = TransformPrototypeIntTest.create_process(  name= 'event_alert_transform',
                                    module='ion.processes.data.transforms.event_alert_transform',
                                    class_name='EventAlertTransform',
                                    configuration= configuration)

        self.assertIsNotNone(pid)

        proc = self.container.proc_manager.procs_by_name[pid]

        #-------------------------------------------------------------------------------------
        # Publish events and make assertions about alerts
        #-------------------------------------------------------------------------------------

        queue = gevent.queue.Queue()

        def event_received(message, headers):
            queue.put(message)

        event_subscriber = EventSubscriber( origin="EventAlertTransform",
                                            event_type="DeviceEvent",
                                            callback=event_received)

        event_subscriber.start()

        # publish event twice

        for i in xrange(2):
            self.event_publisher.publish_event(     event_type='ExampleDetectableEvent',
                                                    origin = "instrument_A",
                                                    ts_created = i,
                                                    voltage = 5,
                                                    telemetry = 10,
                                                    temperature = 20)
            gevent.sleep(4)
            self.assertTrue(queue.empty())

        #publish event the third time

        self.event_publisher.publish_event(     event_type='ExampleDetectableEvent',
                                                origin = "instrument_A",
                                                ts_created = 4,
                                                voltage = 5,
                                                telemetry = 10,
                                                temperature = 20)
        gevent.sleep(4)

        #-------------------------------------------------------------------------------------
        # Make assertions about the alert event published by the EventAlertTransform
        #-------------------------------------------------------------------------------------

        event = queue.get(timeout=10)

        self.assertEquals(event.type_, "DeviceEvent")
        self.assertEquals(event.origin, "EventAlertTransform")

        log.debug("This completes the requirement that the EventAlertTransform publishes \
         an event after a fixed number of events of a particular type are received from some instrument.")

    def test_stream_processing(self):
        '''
        Test that streams are processed by the transforms according to a provided algorithm
        '''

        #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH,
        #todo(contd) and if the word VALUE=<number> exists and that number is less than something

        #todo later on we are going to use complex algorithms to make this prototype powerful

        #-------------------------------------------------------------------------------------
        # Start a subscriber to listen for an alert event from the Stream Alert Transform
        #-------------------------------------------------------------------------------------

        queue = gevent.queue.Queue()

        def event_received(message, headers):
            queue.put(message)

        event_subscriber = EventSubscriber( origin="StreamAlertTransform",
            event_type="DeviceEvent",
            callback=event_received)

        event_subscriber.start()

        #-------------------------------------------------------------------------------------
        # The configuration for the Stream Alert Transform... set up the event types to listen to
        #-------------------------------------------------------------------------------------
        config = {
            'process':{
                'queue_name': 'a_queue',
                'value': 10
            }
        }

        #-------------------------------------------------------------------------------------
        # Create the process
        #-------------------------------------------------------------------------------------
        pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process',
                                                        module='ion.processes.data.transforms.event_alert_transform',
                                                        class_name='StreamAlertTransform',
                                                        configuration= config)

        self.assertIsNotNone(pid)

        #-------------------------------------------------------------------------------------
        # Publish streams and make assertions about alerts
        #-------------------------------------------------------------------------------------
        exchange_name = 'a_queue'
        exchange_point = 'test_exchange'

        xn = self.container.ex_manager.create_xn_queue(exchange_name)
        xp = self.container.ex_manager.create_xp(exchange_point)
        xn.bind('stream_id.data', xp)

        pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')

        message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" + \
                    " will trigger an alert event from the StreamAlertTransform"

        pub.publish(message)

        gevent.sleep(4)

        event = queue.get()
        self.assertEquals(event.type_, "DeviceEvent")
        self.assertEquals(event.origin, "StreamAlertTransform")

        self.purge_queues(exchange_name)

    def purge_queues(self, exchange_name):
        xn = self.container.ex_manager.create_xn_queue(exchange_name)
        xn.purge()

    @staticmethod
    def create_process(name= '', module = '', class_name = '', configuration = None):
        '''
        A helper method to create a process
        '''

        producer_definition = ProcessDefinition(name=name)
        producer_definition.executable = {
            'module':module,
            'class': class_name
        }

        process_dispatcher = ProcessDispatcherServiceClient()

        procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
        pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration)

        return pid
<commit_msg>Added skip if in CEI launch mode<commit_after>#!/usr/bin/env python

'''
@brief Test the new transform prototype against streams and events
@author Swarbhanu Chatterjee
'''

from pyon.ion.transforma import TransformEventListener, TransformEventPublisher, TransformAlgorithm
from pyon.public import log
from pyon.ion.stream import SimpleStreamPublisher
from pyon.util.containers import DotDict
from pyon.util.file_sys import FileSystem
from pyon.util.int_test import IonIntegrationTestCase
from pyon.util.unit_test import IonUnitTestCase
from pyon.event.event import EventPublisher, EventSubscriber
from nose.plugins.attrib import attr
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
from interface.services.cei.iprocess_dispatcher_service import ProcessDispatcherServiceClient
from interface.objects import ProcessDefinition
from mock import Mock, sentinel, patch
import gevent, unittest, os

@attr('INT', group='dm')
class TransformPrototypeIntTest(IonIntegrationTestCase):
    def setUp(self):
        super(TransformPrototypeIntTest, self).setUp()

        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        self.rrc = ResourceRegistryServiceClient()
        self.event_publisher = EventPublisher()

    @attr('LOCOINT')
    @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
    def test_event_processing(self):
        '''
        Test that events are processed by the transforms according to a provided algorithm
        '''

        #-------------------------------------------------------------------------------------
        # Create an event alert transform
        #-------------------------------------------------------------------------------------

        #-------------------------------------------------------------------------------------
        # The configuration for the Event Alert Transform... set up the event types to listen to
        #-------------------------------------------------------------------------------------
        configuration = {
                            'process':{
                                'event_type': 'ExampleDetectableEvent',
                                'max_count': 3,
                                'time_window': 5,
                            }
                        }

        #-------------------------------------------------------------------------------------
        # Create the process
        #-------------------------------------------------------------------------------------
        pid = TransformPrototypeIntTest.create_process(  name= 'event_alert_transform',
                                    module='ion.processes.data.transforms.event_alert_transform',
                                    class_name='EventAlertTransform',
                                    configuration= configuration)

        self.assertIsNotNone(pid)

        proc = self.container.proc_manager.procs_by_name[pid]

        #-------------------------------------------------------------------------------------
        # Publish events and make assertions about alerts
        #-------------------------------------------------------------------------------------

        queue = gevent.queue.Queue()

        def event_received(message, headers):
            queue.put(message)

        event_subscriber = EventSubscriber( origin="EventAlertTransform",
                                            event_type="DeviceEvent",
                                            callback=event_received)

        event_subscriber.start()

        # publish event twice

        for i in xrange(2):
            self.event_publisher.publish_event(     event_type='ExampleDetectableEvent',
                                                    origin = "instrument_A",
                                                    ts_created = i,
                                                    voltage = 5,
                                                    telemetry = 10,
                                                    temperature = 20)
            gevent.sleep(4)
            self.assertTrue(queue.empty())

        #publish event the third time

        self.event_publisher.publish_event(     event_type='ExampleDetectableEvent',
                                                origin = "instrument_A",
                                                ts_created = 4,
                                                voltage = 5,
                                                telemetry = 10,
                                                temperature = 20)
        gevent.sleep(4)

        #-------------------------------------------------------------------------------------
        # Make assertions about the alert event published by the EventAlertTransform
        #-------------------------------------------------------------------------------------

        event = queue.get(timeout=10)

        self.assertEquals(event.type_, "DeviceEvent")
        self.assertEquals(event.origin, "EventAlertTransform")

        log.debug("This completes the requirement that the EventAlertTransform publishes \
         an event after a fixed number of events of a particular type are received from some instrument.")

    @attr('LOCOINT')
    @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
    def test_stream_processing(self):
        '''
        Test that streams are processed by the transforms according to a provided algorithm
        '''

        #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH,
        #todo(contd) and if the word VALUE=<number> exists and that number is less than something

        #todo later on we are going to use complex algorithms to make this prototype powerful

        #-------------------------------------------------------------------------------------
        # Start a subscriber to listen for an alert event from the Stream Alert Transform
        #-------------------------------------------------------------------------------------

        queue = gevent.queue.Queue()

        def event_received(message, headers):
            queue.put(message)

        event_subscriber = EventSubscriber( origin="StreamAlertTransform",
            event_type="DeviceEvent",
            callback=event_received)

        event_subscriber.start()

        #-------------------------------------------------------------------------------------
        # The configuration for the Stream Alert Transform... set up the event types to listen to
        #-------------------------------------------------------------------------------------
        config = {
            'process':{
                'queue_name': 'a_queue',
                'value': 10
            }
        }

        #-------------------------------------------------------------------------------------
        # Create the process
        #-------------------------------------------------------------------------------------
        pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process',
                                                        module='ion.processes.data.transforms.event_alert_transform',
                                                        class_name='StreamAlertTransform',
                                                        configuration= config)

        self.assertIsNotNone(pid)

        #-------------------------------------------------------------------------------------
        # Publish streams and make assertions about alerts
        #-------------------------------------------------------------------------------------
        exchange_name = 'a_queue'
        exchange_point = 'test_exchange'

        xn = self.container.ex_manager.create_xn_queue(exchange_name)
        xp = self.container.ex_manager.create_xp(exchange_point)
        xn.bind('stream_id.data', xp)

        pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')

        message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" + \
                    " will trigger an alert event from the StreamAlertTransform"

        pub.publish(message)

        gevent.sleep(4)

        event = queue.get()
        self.assertEquals(event.type_, "DeviceEvent")
        self.assertEquals(event.origin, "StreamAlertTransform")

        self.purge_queues(exchange_name)

    def purge_queues(self, exchange_name):
        xn = self.container.ex_manager.create_xn_queue(exchange_name)
        xn.purge()

    @staticmethod
    def create_process(name= '', module = '', class_name = '', configuration = None):
        '''
        A helper method to create a process
        '''

        producer_definition = ProcessDefinition(name=name)
        producer_definition.executable = {
            'module':module,
            'class': class_name
        }

        process_dispatcher = ProcessDispatcherServiceClient()

        procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
        pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration)

        return pid
<|endoftext|>