<commit_before>#!/usr/bin/env python
"""Handle mouse controlled small target stimulus (client-side)"""

# Copyright (c) 2002-2003 Andrew Straw.  Distributed under the terms
# of the GNU Lesser General Public License (LGPL).

import VisionEgg, string
__version__ = VisionEgg.release_name
__cvs__ = string.split('$Revision$')[1]
__date__ = string.join(string.split('$Date$')[1:3], ' ')
__author__ = 'Andrew Straw <astraw@users.sourceforge.net>'

import sys, os
import Tkinter
import VisionEgg.PyroApps.EPhysGUIUtils as client_utils

def get_control_list():
    return [("mouse_target_server",MouseTargetControlFrame,MouseTargetControlFrame.title)]

class MouseTargetMetaParameters:
    def __init__(self):
        # colors
        self.color = (0.0, 0.0, 0.0, 1.0)
        self.bgcolor = (1.0, 1.0, 1.0, 0.0)

class MouseTargetControlFrame(client_utils.StimulusControlFrame):
    title = "Mouse Controlled Moving Target"
    def __init__(self, master=None, suppress_go_buttons=0,**kw):
        apply(client_utils.StimulusControlFrame.__init__,(self,
                                                          master,
                                                          suppress_go_buttons,
                                                          MouseTargetControlFrame.title,
                                                          MouseTargetMetaParameters),kw)

        param_frame = self.param_frame # shorthand for self.param_frame created in base class

        # Allow columns to expand
        param_frame.columnconfigure(0,weight=1)
        param_frame.columnconfigure(1,weight=1)

        pf_row = 0
        if 0:
            Tkinter.Label(param_frame,text="Start X (pixels):").grid(row=pf_row,column=0)
            self.start_x_tk_var = Tkinter.DoubleVar()
            self.start_x_tk_var.set(self.meta_params.start_x)
            self.make_callback_entry(textvariable=self.start_x_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Start X"] = ("start_x",self.start_x_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Start Y (pixels):").grid(row=pf_row,column=0)
            self.start_y_tk_var = Tkinter.DoubleVar()
            self.start_y_tk_var.set(self.meta_params.start_y)
            self.make_callback_entry(textvariable=self.start_y_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Start Y"] = ("start_y",self.start_y_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Velocity (pixels/sec):").grid(row=pf_row,column=0)
            self.velocity_tk_var = Tkinter.DoubleVar()
            self.velocity_tk_var.set(self.meta_params.velocity_pps)
            self.make_callback_entry(textvariable=self.velocity_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Velocity"] = ("velocity_pps",self.velocity_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Direction (degrees):").grid(row=pf_row,column=0)
            self.direction_tk_var = Tkinter.DoubleVar()
            self.direction_tk_var.set(self.meta_params.direction_deg)
            self.make_callback_entry(textvariable=self.direction_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Direction"] = ("direction_deg",self.direction_tk_var)

            pf_row += 1
        Tkinter.Label(param_frame,text="Color:").grid(row=pf_row,column=0)
        self.color_tk_var = Tkinter.StringVar()
        self.color_tk_var.set("black on white")
        bar = Tkinter.Menubutton(param_frame, textvariable=self.color_tk_var, relief=Tkinter.RAISED)
        bar.grid(row=pf_row, column=2, sticky=Tkinter.W+Tkinter.E, pady=2, padx=2)
        bar.menu = Tkinter.Menu(bar,tearoff=0)
        bar.menu.add_radiobutton(label="white on black",
                                 value="white on black",
                                 variable=self.color_tk_var,
                                 command=self.send_values)
        bar.menu.add_radiobutton(label="black on white",
                                 value="black on white",
                                 variable=self.color_tk_var,
                                 command=self.send_values)
        bar['menu'] = bar.menu

        if 0:
            pf_row += 1
            Tkinter.Label(param_frame,text="Orientation (degrees):").grid(row=pf_row,column=0)
            self.ortho_tk_var = Tkinter.StringVar()
            self.ortho_tk_var.set("ortho")

            ortho = Tkinter.Radiobutton( param_frame, text="Orthogonal",
                                        variable=self.ortho_tk_var, value="ortho")
            ortho.grid(row=pf_row,column=3)
            manual = Tkinter.Radiobutton( param_frame, #text="Orthogonal",
                                        variable=self.ortho_tk_var, value="manual")
            manual.grid(row=pf_row,column=1)
            self.orient_tk_var = Tkinter.DoubleVar()
            self.orient_tk_var.set(self.meta_params.orientation_deg)
            manual_entry = Tkinter.Entry( param_frame,
                                        textvariable=self.orient_tk_var,
                                        width=self.entry_width )
            manual_entry.grid(row=pf_row,column=2)
            self.loopable_variables["Orientation"] = ("orientation_deg",self.orient_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Width (pixels):").grid(row=pf_row,column=0)
            self.width_tk_var = Tkinter.DoubleVar()
            self.width_tk_var.set(self.meta_params.width)
            self.make_callback_entry(textvariable=self.width_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Width"] = ("width",self.width_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Height (pixels):").grid(row=pf_row,column=0)
            self.height_tk_var = Tkinter.DoubleVar()
            self.height_tk_var.set(self.meta_params.height)
            self.make_callback_entry(textvariable=self.height_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Height"] = ("height",self.height_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Pre stimulus duration (sec):").grid(row=pf_row,column=0)
            self.prestim_dur_tk_var = Tkinter.DoubleVar()
            self.prestim_dur_tk_var.set(self.meta_params.pre_stim_sec)
            self.make_callback_entry(textvariable=self.prestim_dur_tk_var).grid(row=pf_row,column=2)

            pf_row += 1
            Tkinter.Label(param_frame,text="Stimulus duration (sec):").grid(row=pf_row,column=0)
            self.stim_dur_tk_var = Tkinter.DoubleVar()
            self.stim_dur_tk_var.set(self.meta_params.stim_sec)
            self.make_callback_entry(textvariable=self.stim_dur_tk_var).grid(row=pf_row,column=2)
            self.loopable_variables["Duration"] = ("stim_sec",self.stim_dur_tk_var)

            pf_row += 1
            Tkinter.Label(param_frame,text="Post stimulus duration (sec):").grid(row=pf_row,column=0)
            self.poststim_dur_tk_var = Tkinter.DoubleVar()
            self.poststim_dur_tk_var.set(self.meta_params.post_stim_sec)
            self.make_callback_entry(textvariable=self.poststim_dur_tk_var).grid(row=pf_row,column=2)

    def get_shortname(self):
        return "mouse_target"

    def update_tk_vars(self):
        if 0:
            self.start_x_tk_var.set( self.meta_params.start_x )
            self.start_y_tk_var.set( self.meta_params.start_y )
            self.velocity_tk_var.set( self.meta_params.velocity_pps )
            self.direction_tk_var.set( self.meta_params.direction_deg )
            self.orient_tk_var.set( self.meta_params.orientation_deg )
            self.width_tk_var.set( self.meta_params.width )
            self.height_tk_var.set( self.meta_params.height )
            self.prestim_dur_tk_var.set( self.meta_params.pre_stim_sec )
            self.stim_dur_tk_var.set( self.meta_params.stim_sec )
            self.poststim_dur_tk_var.set( self.meta_params.post_stim_sec )

        if self.meta_params.color == (0.0,0.0,0.0,1.0) and self.meta_params.bgcolor == (1.0,1.0,1.0,0.0):
            self.color_tk_var.set( "black on white" )
        elif self.meta_params.color == (1.0,1.0,1.0,1.0) and self.meta_params.bgcolor == (0.0,0.0,0.0,0.0):
            self.color_tk_var.set( "white on black" )
        else:
            raise RuntimeError("Cannot set tk variable for color")

    def send_values(self,dummy_arg=None):
        if self.color_tk_var.get() == "black on white":
            self.meta_params.color = (0.0,0.0,0.0,1.0)
            self.meta_params.bgcolor = (1.0,1.0,1.0,0.0)
        elif self.color_tk_var.get() == "white on black":
            self.meta_params.color = (1.0,1.0,1.0,1.0)
            self.meta_params.bgcolor = (0.0,0.0,0.0,0.0)

        if 0:
            self.meta_params.start_x = self.start_x_tk_var.get()
            self.meta_params.start_y = self.start_y_tk_var.get()
            self.meta_params.velocity_pps = self.velocity_tk_var.get()
            self.meta_params.direction_deg = self.direction_tk_var.get()

            if self.ortho_tk_var.get():
                self.meta_params.orientation_deg = self.meta_params.direction_deg
            else:
                self.meta_params.orientation_deg = self.orient_tk_var.get()
            self.meta_params.width = self.width_tk_var.get()
            self.meta_params.height = self.height_tk_var.get()
            self.meta_params.pre_stim_sec = self.prestim_dur_tk_var.get()
            self.meta_params.stim_sec = self.stim_dur_tk_var.get()
            self.meta_params.post_stim_sec = self.poststim_dur_tk_var.get()
        if self.connected:
            self.meta_controller.set_parameters( self.meta_params )

    def get_duration_sec(self):
        if 0:
            self.meta_params.pre_stim_sec = self.prestim_dur_tk_var.get()
            self.meta_params.stim_sec = self.stim_dur_tk_var.get()
            self.meta_params.post_stim_sec = self.poststim_dur_tk_var.get()
            return self.meta_params.pre_stim_sec + self.meta_params.stim_sec + self.meta_params.post_stim_sec
        return 0.0

if __name__=='__main__':
    frame = MouseTargetControlFrame()
    frame.pack(expand=1,fill=Tkinter.BOTH)
    frame.winfo_toplevel().title("%s"%(os.path.basename(os.path.splitext(sys.argv[0])[0]),))
    frame.mainloop()
<commit_msg>Trivial: removed unused codepaths<commit_after>#!/usr/bin/env python
"""Handle mouse controlled small target stimulus (client-side)"""

# Copyright (c) 2002-2003 Andrew Straw.  Distributed under the terms
# of the GNU Lesser General Public License (LGPL).

import VisionEgg, string
__version__ = VisionEgg.release_name
__cvs__ = string.split('$Revision$')[1]
__date__ = string.join(string.split('$Date$')[1:3], ' ')
__author__ = 'Andrew Straw <astraw@users.sourceforge.net>'

import sys, os
import Tkinter
import VisionEgg.PyroApps.EPhysGUIUtils as client_utils

def get_control_list():
    return [("mouse_target_server",MouseTargetControlFrame,MouseTargetControlFrame.title)]

class MouseTargetMetaParameters:
    def __init__(self):
        # colors
        self.color = (0.0, 0.0, 0.0, 1.0)
        self.bgcolor = (1.0, 1.0, 1.0, 0.0)

class MouseTargetControlFrame(client_utils.StimulusControlFrame):
    title = "Mouse Controlled Moving Target"
    def __init__(self, master=None, suppress_go_buttons=0,**kw):
        apply(client_utils.StimulusControlFrame.__init__,(self,
                                                          master,
                                                          suppress_go_buttons,
                                                          MouseTargetControlFrame.title,
                                                          MouseTargetMetaParameters),kw)

        param_frame = self.param_frame # shorthand for self.param_frame created in base class

        # Allow columns to expand
        param_frame.columnconfigure(0,weight=1)
        param_frame.columnconfigure(1,weight=1)

        pf_row = 0
        Tkinter.Label(param_frame,text="Color:").grid(row=pf_row,column=0)
        self.color_tk_var = Tkinter.StringVar()
        self.color_tk_var.set("black on white")
        bar = Tkinter.Menubutton(param_frame, textvariable=self.color_tk_var, relief=Tkinter.RAISED)
        bar.grid(row=pf_row, column=2, sticky=Tkinter.W+Tkinter.E, pady=2, padx=2)
        bar.menu = Tkinter.Menu(bar,tearoff=0)
        bar.menu.add_radiobutton(label="white on black",
                                 value="white on black",
                                 variable=self.color_tk_var,
                                 command=self.send_values)
        bar.menu.add_radiobutton(label="black on white",
                                 value="black on white",
                                 variable=self.color_tk_var,
                                 command=self.send_values)
        bar['menu'] = bar.menu

    def get_shortname(self):
        return "mouse_target"

    def update_tk_vars(self):
        if self.meta_params.color == (0.0,0.0,0.0,1.0) and self.meta_params.bgcolor == (1.0,1.0,1.0,0.0):
            self.color_tk_var.set( "black on white" )
        elif self.meta_params.color == (1.0,1.0,1.0,1.0) and self.meta_params.bgcolor == (0.0,0.0,0.0,0.0):
            self.color_tk_var.set( "white on black" )
        else:
            raise RuntimeError("Cannot set tk variable for color")

    def send_values(self,dummy_arg=None):
        if self.color_tk_var.get() == "black on white":
            self.meta_params.color = (0.0,0.0,0.0,1.0)
            self.meta_params.bgcolor = (1.0,1.0,1.0,0.0)
        elif self.color_tk_var.get() == "white on black":
            self.meta_params.color = (1.0,1.0,1.0,1.0)
            self.meta_params.bgcolor = (0.0,0.0,0.0,0.0)

        if self.connected:
            self.meta_controller.set_parameters( self.meta_params )

    def get_duration_sec(self):
        return 0.0

if __name__=='__main__':
    frame = MouseTargetControlFrame()
    frame.pack(expand=1,fill=Tkinter.BOTH)
    frame.winfo_toplevel().title("%s"%(os.path.basename(os.path.splitext(sys.argv[0])[0]),))
    frame.mainloop()
<|endoftext|>


<commit_before>from ..utils import TranspileTestCase

class ClassTests(TranspileTestCase):
    def test_minimal(self):
        self.assertCodeExecution("""
            class MyClass:
                pass

            obj = MyClass()
            """, run_in_function=False)

    def test_simple(self):
        self.assertCodeExecution("""
            class MyClass:
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_subclass_object(self):
        self.assertCodeExecution("""
            class MyClass(object):
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_method_override(self):
        self.assertCodeExecution("""
            class MyObject:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Myobject instance %s" % self.x

            obj = MyObject(37)

            print(obj)
            """, run_in_function=False)

    def test_subclass(self):
        self.assertCodeExecution("""
            class MyBase:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Mybase instance %s" % self.x

                def first(self):
                    return self.x * 2

            class MyObject(MyBase):
                def __init__(self, x, y):
                    super().__init__(x)
                    self.y = y

                def __str__(self):
                    return "Myobject instance %s, %s" % (self.x, self.y)

                def second(self):
                    return self.x * self.y

            obj = MyObject(37, 42)

            print(obj)
            print(obj.x)
            print(obj.first())
            print(obj.y)
            print(obj.second())
            """)

    def test_subclass_2_clause_super(self):
        self.assertCodeExecution("""
            class MyBase:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Mybase instance %s" % self.x

                def first(self):
                    return self.x * 2

            class MyObject(MyBase):
                def __init__(self, x, y):
                    super(MyObject, self).__init__(x)
                    self.y = y

                def __str__(self):
                    return "Myobject instance %s, %s" % (self.x, self.y)

                def second(self):
                    return self.x * self.y

            obj = MyObject(37, 42)

            print(obj)
            print(obj.x)
            print(obj.first())
            print(obj.y)
            print(obj.second())
            """)

    def test_redefine(self):
        self.assertCodeExecution("""
            class MyClass:
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

                def stuff(self, delta):
                    print("Redefined DELTA: ", delta)
                    return self.value + delta * 2

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_overwrite_class_attributes(self):
        self.assertCodeExecution("""
            class MyClass:
                x = 1

            print(MyClass.x)

            inst = MyClass()
            print(inst.x)

            inst.x = 123
            print(inst.x)

            print(MyClass.x)

            inst2 = MyClass()
            inst2.x = 5
            print(inst2.x)
            """, run_in_function=False)

class ClassMethodTests(TranspileTestCase):
    @expectedFailure
    def test_classmethod(self):
        self.assertCodeExecution("""
            class MyClass:
                @classmethod
                def foo(cls, arg1, arg2):
                    print("This is a classmethod on ", cls, arg1, arg2)

            obj = MyClass()
            obj.foo(1, 2)
            MyClass.foo(3, 4)
            """, run_in_function=False)

class StaticMethodTests(TranspileTestCase):
    @expectedFailure
    def test_staticmethod(self):
        self.assertCodeExecution("""
            class MyClass:
                @staticmethod
                def foo(arg1, arg2):
                    print("This is a staticmethod ", arg1, arg2)

            obj = MyClass()
            obj.foo(1, 2)
            MyClass.foo(3, 4)
            """, run_in_function=False)
<commit_msg>Fixed flake8 failures.<commit_after>from unittest import expectedFailure

from ..utils import TranspileTestCase

class ClassTests(TranspileTestCase):
    def test_minimal(self):
        self.assertCodeExecution("""
            class MyClass:
                pass

            obj = MyClass()
            """, run_in_function=False)

    def test_simple(self):
        self.assertCodeExecution("""
            class MyClass:
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_subclass_object(self):
        self.assertCodeExecution("""
            class MyClass(object):
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_method_override(self):
        self.assertCodeExecution("""
            class MyObject:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Myobject instance %s" % self.x

            obj = MyObject(37)

            print(obj)
            """, run_in_function=False)

    def test_subclass(self):
        self.assertCodeExecution("""
            class MyBase:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Mybase instance %s" % self.x

                def first(self):
                    return self.x * 2

            class MyObject(MyBase):
                def __init__(self, x, y):
                    super().__init__(x)
                    self.y = y

                def __str__(self):
                    return "Myobject instance %s, %s" % (self.x, self.y)

                def second(self):
                    return self.x * self.y

            obj = MyObject(37, 42)

            print(obj)
            print(obj.x)
            print(obj.first())
            print(obj.y)
            print(obj.second())
            """)

    def test_subclass_2_clause_super(self):
        self.assertCodeExecution("""
            class MyBase:
                def __init__(self, x):
                    self.x = x

                def __str__(self):
                    return "Mybase instance %s" % self.x

                def first(self):
                    return self.x * 2

            class MyObject(MyBase):
                def __init__(self, x, y):
                    super(MyObject, self).__init__(x)
                    self.y = y

                def __str__(self):
                    return "Myobject instance %s, %s" % (self.x, self.y)

                def second(self):
                    return self.x * self.y

            obj = MyObject(37, 42)

            print(obj)
            print(obj.x)
            print(obj.first())
            print(obj.y)
            print(obj.second())
            """)

    def test_redefine(self):
        self.assertCodeExecution("""
            class MyClass:
                def __init__(self, val):
                    print("VAL: ", val)
                    self.value = val

                def stuff(self, delta):
                    print("DELTA: ", delta)
                    return self.value + delta

                def stuff(self, delta):
                    print("Redefined DELTA: ", delta)
                    return self.value + delta * 2

            obj = MyClass(4)
            obj.stuff(5)
            """, run_in_function=False)

    def test_overwrite_class_attributes(self):
        self.assertCodeExecution("""
            class MyClass:
                x = 1

            print(MyClass.x)

            inst = MyClass()
            print(inst.x)

            inst.x = 123
            print(inst.x)

            print(MyClass.x)

            inst2 = MyClass()
            inst2.x = 5
            print(inst2.x)
            """, run_in_function=False)

class ClassMethodTests(TranspileTestCase):
    @expectedFailure
    def test_classmethod(self):
        self.assertCodeExecution("""
            class MyClass:
                @classmethod
                def foo(cls, arg1, arg2):
                    print("This is a classmethod on ", cls, arg1, arg2)

            obj = MyClass()
            obj.foo(1, 2)
            MyClass.foo(3, 4)
            """, run_in_function=False)

class StaticMethodTests(TranspileTestCase):
    @expectedFailure
    def test_staticmethod(self):
        self.assertCodeExecution("""
            class MyClass:
                @staticmethod
                def foo(arg1, arg2):
                    print("This is a staticmethod ", arg1, arg2)

            obj = MyClass()
            obj.foo(1, 2)
            MyClass.foo(3, 4)
            """, run_in_function=False)
<|endoftext|>


<commit_before>import argparse
import re
import tempfile
import time

import simulation.optimization.cost_function
import simulation.optimization.job
import simulation.model.constants

import measurements.all.pw.data

import util.batch.universal.system
import util.logging
logger = util.logging.logger

def save(measurements_collection, model_names=None, cost_function_classes=None, eval_f=True, eval_df=False):
    for cost_function in simulation.optimization.cost_function.iterator(measurements_collection, model_names=model_names, cost_function_classes=cost_function_classes):
        if eval_f and not cost_function.f_available():
            logger.info('Saving cost function f value in {}'.format(cost_function.model.parameter_set_dir))
            cost_function.f()
        if eval_df and not cost_function.df_available():
            logger.info('Saving cost function df value in {}'.format(cost_function.model.parameter_set_dir))
            cost_function.df()

def save_for_all_measurements(max_box_distance_to_water=float('inf'), min_measurements_correlations=float('inf'), model_names=None, cost_function_classes=None, eval_f=True, eval_df=False):
    measurements_collection = measurements.all.pw.data.all_measurements(max_box_distance_to_water=max_box_distance_to_water, min_measurements_correlations=min_measurements_correlations)
    save(measurements_collection, model_names=model_names, cost_function_classes=cost_function_classes, eval_f=eval_f, eval_df=eval_df)

def save_for_all_measurements_with_jobs(max_box_distance_to_water=float('inf'), min_measurements_correlations=float('inf'), model_names=None, cost_function_classes=None, eval_f=True, eval_df=False, node_kind='clexpress'):
    measurements_collection = measurements.all.pw.data.all_measurements(max_box_distance_to_water=max_box_distance_to_water, min_measurements_correlations=min_measurements_correlations)
    nodes_setup = util.batch.universal.system.NodeSetup(memory=50, node_kind=node_kind, nodes=1, cpus=1, total_cpus_max=1, walltime=1)
    for cost_function in simulation.optimization.cost_function.iterator(measurements_collection, model_names=model_names, cost_function_classes=cost_function_classes):
        if (eval_f and not cost_function.f_available()) or (eval_df and not cost_function.df_available()):
                output_dir = tempfile.TemporaryDirectory(dir=simulation.model.constants.DATABASE_TMP_DIR, prefix='save_value_cost_function_tmp_').name
                ints_str = ','.join(re.findall('\d+', cost_function.model.parameter_set_dir)[-3:])
                job_name = '{cost_function_name}:{model_name}:{ints_str}'.format(cost_function_name=cost_function.name, model_name=cost_function.model.model_options.model_name, ints_str=ints_str)
                job_options = {'name': job_name, 'node_setup': nodes_setup}
                with simulation.optimization.job.CostFunctionJob(output_dir, cost_function.name, cost_function.model.model_options, max_box_distance_to_water=max_box_distance_to_water, min_measurements_correlations=min_measurements_correlations, eval_f=eval_f, eval_df=eval_df, job_options=job_options) as cost_function_job:
                    cost_function_job.start()
                time.sleep(10)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Calculating cost function values.')
    parser.add_argument('-m', '--max_box_distance_to_water', type=int, default=float('inf'), help='The maximal distance to water boxes to accept measurements.')
    parser.add_argument('-c', '--min_measurements_correlations', type=int, default=float('inf'), help='The minimal number of measurements used to calculate correlations.')
    parser.add_argument('-j', '--as_jobs', action='store_true', help='Run as jobs.')
    parser.add_argument('-n', '--node_kind', default='clexpress', help='Node kind to use for the jobs.')
    parser.add_argument('--DF', action='store_true', help='Eval (also) DF.')
    parser.add_argument('-d', '--debug_level', choices=util.logging.LEVELS, default='INFO', help='Print debug infos low to passed level.')
    args = parser.parse_args()

    with util.logging.Logger(level=args.debug_level):
        if args.as_jobs:
            save_for_all_measurements_with_jobs(max_box_distance_to_water=args.max_box_distance_to_water, min_measurements_correlations=args.min_measurements_correlations, eval_f=True, eval_df=args.DF, node_kind=args.node_kind)
        else:
            save_for_all_measurements(max_box_distance_to_water=args.max_box_distance_to_water, min_measurements_correlations=args.min_measurements_correlations, eval_f=True, eval_df=args.DF)
<commit_msg>ENH: simulation.optimization.save: updated to new model and measurement style<commit_after>import argparse
import re
import tempfile
import time

import numpy as np

import simulation.optimization.cost_function
import simulation.optimization.job
import simulation.model.constants
import simulation.model.options

import measurements.all.pw.data

import util.batch.universal.system
import util.logging
logger = util.logging.logger

def save(cost_functions, model_names=None, eval_f=True, eval_df=False):
    for cost_function in simulation.optimization.cost_function.iterator(cost_functions, model_names=model_names):
        if eval_f and not cost_function.f_available():
            logger.info('Saving cost function f value in {}'.format(cost_function.model.parameter_set_dir))
            cost_function.f()
        if eval_df and not cost_function.df_available():
            logger.info('Saving cost function df value in {}'.format(cost_function.model.parameter_set_dir))
            cost_function.df()

def save_for_all_measurements(max_box_distance_to_water_list=None, min_measurements_correlation_list=None, cost_function_classes=None, model_names=None, eval_f=True, eval_df=False):
    model_options = simulation.model.options.ModelOptions()
    model_options.spinup_options = {'years':1, 'tolerance':0.0, 'combination':'or'}
    cost_functions = simulation.optimization.cost_function.cost_functions_for_all_measurements(max_box_distance_to_water_list=max_box_distance_to_water_list, min_measurements_correlation_list=min_measurements_correlation_list, cost_function_classes=cost_function_classes, model_options=model_options)
    save(cost_functions, model_names=model_names, eval_f=eval_f, eval_df=eval_df)

if __name__ == "__main__":
    ## parse args
    parser = argparse.ArgumentParser(description='Calculating cost function values.')
    parser.add_argument('--min_measurements_correlation_list', type=int, default=None, nargs='+', help='The minimal number of measurements used to calculate correlations.')
    parser.add_argument('--max_box_distance_to_water_list', type=int, default=None, nargs='+', help='The maximal distances to water boxes to accept measurements.')
    parser.add_argument('--cost_function_list', type=str, default=None, nargs='+', help='The cost function to evaluate.')
    parser.add_argument('--DF', action='store_true', help='Eval (also) DF.')
    parser.add_argument('--debug_level', choices=util.logging.LEVELS, default='INFO', help='Print debug infos low to passed level.')
    args = parser.parse_args()

    ## max_box_distance_to_water
    max_box_distance_to_water = np.array(args.max_box_distance_to_water_list)
    add_float = np.any(max_box_distance_to_water < 0)
    max_box_distance_to_water = max_box_distance_to_water[max_box_distance_to_water >= 0]
    max_box_distance_to_water = np.unique(max_box_distance_to_water)
    max_box_distance_to_water_list = max_box_distance_to_water.tolist()
    if add_float:
        max_box_distance_to_water_list = max_box_distance_to_water_list + [float('inf')]

    ## cost_function_classes
    if args.cost_function_list is None:
        cost_function_classes = None
    else:
        cost_function_classes = [getattr(simulation.optimization.cost_function, cost_function_name) for cost_function_name in args.cost_function_list]

    ## run
    with util.logging.Logger(level=args.debug_level):
        save_for_all_measurements(max_box_distance_to_water_list=max_box_distance_to_water_list, min_measurements_correlation_list=args.min_measurements_correlation_list, cost_function_classes=cost_function_classes, eval_f=True, eval_df=args.DF)
        logger.info('Finished.')
<|endoftext|>


<commit_before>#!/bin/python

import argparse
import sys
from math import floor
from matplotlib.colors import cnames

import classifiers as cl
import utils
from spiral import *

parser = argparse.ArgumentParser(description='Classify some data.')

parser.add_argument('-k', type=int, default=1, help='k Nearest Neighbor classifier')
parser.add_argument('-d', '--distance', required=True, choices=['euclidian', 'hamming+'], help='Distance metric algorithm')
parser.add_argument('-t', '--training_set', type=str, help='Filename of the training set, must be a CSV file')
parser.add_argument('-i', '--input', type=str, help='Filename of the input to classify, must be a CSV file')
parser.add_argument('-o', '--output', required=True, type=str, help='Output file')
parser.add_argument('-c', '--category', type=int, help='Column of the class')
parser.add_argument('-I', '--ignore', action='append', type=int, help='Columns to be ignored')
parser.add_argument('-s', '--spiral', type=str, choices=['single', 'double'], help='Number of spirals')
parser.add_argument('-g', '--grid_size', type=int, help='Grid size (used for spirals only)')
parser.add_argument('-n', '--noise', type=int, default=0, help='Noise for the spiral(s).')
parser.add_argument('-v', '--verbose', action='store_true', help='Print additional statistics')
parser.add_argument('-p', '--plot', action='store_true', help='Plot the classified patterns (if 2D)')
parser.add_argument('-l', '--slice', type=float, help='Automatically pick the given approximate percentage of the input set to use as the training set')
parser.add_argument('-S', '--save_image', action='store_true', help='Save the resulting spiral as an image')

args = parser.parse_args()

if not args.output or\
  (not args.spiral and args.category == None) or\
  (not args.spiral and not args.input) or\
  (not args.spiral and args.noise) or\
  (args.spiral and (args.training_set or args.input)) or\
  (args.spiral and not args.grid_size) or\
  (args.input and not args.training_set and not args.slice) or\
  (args.training_set and args.slice):

   parser.print_help()
   sys.exit(0)

training_header = []
training_set = []
if args.training_set:
    data = utils.load(args.training_set)
    training_header = data[0]
    training_set = data[1:]

input_set = []
if args.input:
    data = utils.load(args.input)
    if not args.training_set:
        training_header = data[0]
    input_set = data[1:]

distance_function = None
if args.distance == 'euclidian':
    distance_function = cl.euclidian_dist
else:
    distance_function = cl.hamming_dist

if args.slice:
    num_picked_entries = floor(len(input_set) * args.slice / 100)
    num_entries = len(input_set)
    i = 0
    picked_indexes = set()
    while len(training_set) < num_picked_entries:
        if i >= num_entries:
            i = 1
        training_set.append(input_set[i])
        picked_indexes.add(i)
        i += 2

    for i in range(num_entries - 1, -1, -1):
        if i in picked_indexes:
            del input_set[i]

if args.ignore:
    args.ignore.sort(reverse=True)
    for entry in training_set:
        for index in args.ignore:
            del entry[index]

    for entry in input_set:
        for index in args.ignore:
            del entry[index]

    for index in args.ignore:
        if index < args.category:
            args.category -= 1
        del training_header[index]

target_file = args.output

output = []
if not args.spiral:
    hits = 0
    fails = 0
    output.append(training_header)
    for entry in input_set:
        prepared_entry = utils.without_column(entry, args.category)
        class_value = cl.kNN(training_set, prepared_entry, distance_function, \
                             args.category, args.k)
        output.append(prepared_entry + [class_value])
        if entry[args.category] != None:
            if entry[args.category] == class_value:
                hits += 1
            else:
                fails += 1

    utils.save(target_file, output)
    if args.verbose and hits + fails > 0:
        print("Hits: %i" % (hits))
        print("Fails: %i" % (fails))
        print("Precision: %f%%" % (100 * hits / (hits + fails)))

    if args.plot and len(training_set) == 3:
        import plotter as pl

        allcolors = [color for color in cnames]
        categories = list(set([x[args.category] for x in training_set]))

        pl.plot(
            output[:1],
            output[2],
            training_set[:1],
            training_set[:2],
            utils.without_column(training_header, args.category),
            [allcolors[i] for i in range(len(categories))]
        )
else:
    neighborhood = 2 * args.grid_size + 50
    if args.spiral == "single":
        spiral = single_spiral(args.grid_size, args.noise)
        size = args.grid_size
        spiral_points = [(spiral[i][0] + size, spiral[i][1] + size, 0) \
                         for i in range(len(spiral)) if not (i % 2)] + \
                        [(spiral[i][0] + size, spiral[i][1] + size, 1) \
                         for i in range(len(spiral)) if (i % 2)]
    else:
        spirals = double_spiral(args.grid_size, args.noise)
        spiral_points = [(s[0] + args.grid_size, s[1] + args.grid_size, 0) for s in spirals[0]]\
                      + [(s[0] + args.grid_size, s[1] + args.grid_size, 1) for s in spirals[1]]

    red_points = []
    blue_points = []
    for x in range(neighborhood):
        for y in range(neighborhood):
            entry = (x, y)
            class_value = cl.kNN(spiral_points, entry, distance_function, \
                                 2, args.k)
            if class_value == 0:
                red_points.append(entry)
            else:
                blue_points.append(entry)

    first = [(255, 0, 0) for point in red_points]
    second = [(0, 0, 255) for point in blue_points]

    if args.save_image:
        import imagesaver
        imagesaver.save(red_points + blue_points, first + second, neighborhood, args.output)

    if args.plot:
        import plotter as pl

        output = red_points + blue_points
        result = first + second

        pl.plot(
            output,
            result,
            spiral_points[:1],
            [(255, 0, 0) for point in spiral_points if point[2] == 0] + \
            [(0, 0, 255) for point in spiral_points if point[2] == 1],
            ['x', 'y'],
            {(255, 0, 0) : 'red', (0, 0, 255) : 'blue'}
        )<commit_msg>fixed colors of charts<commit_after>#!/bin/python

import argparse
import sys
from math import floor
from matplotlib.colors import cnames

import classifiers as cl
import utils
from spiral import *

parser = argparse.ArgumentParser(description='Classify some data.')

parser.add_argument('-k', type=int, default=1, help='k Nearest Neighbor classifier')
parser.add_argument('-d', '--distance', required=True, choices=['euclidian', 'hamming+'], help='Distance metric algorithm')
parser.add_argument('-t', '--training_set', type=str, help='Filename of the training set, must be a CSV file')
parser.add_argument('-i', '--input', type=str, help='Filename of the input to classify, must be a CSV file')
parser.add_argument('-o', '--output', required=True, type=str, default=out, help='Output file')
parser.add_argument('-c', '--category', type=int, help='Column of the class')
parser.add_argument('-I', '--ignore', action='append', type=int, help='Columns to be ignored')
parser.add_argument('-s', '--spiral', type=str, choices=['single', 'double'], help='Number of spirals')
parser.add_argument('-g', '--grid_size', type=int, help='Grid size (used for spirals only)')
parser.add_argument('-n', '--noise', type=int, default=0, help='Noise for the spiral(s).')
parser.add_argument('-v', '--verbose', action='store_true', help='Print additional statistics')
parser.add_argument('-p', '--plot', action='store_true', help='Plot the classified patterns (if 2D)')
parser.add_argument('-l', '--slice', type=float, help='Automatically pick the given approximate percentage of the input set to use as the training set')
parser.add_argument('-S', '--save_image', action='store_true', help='Save the resulting spiral as an image')

args = parser.parse_args()

if not args.output or\
  (not args.spiral and args.category == None) or\
  (not args.spiral and not args.input) or\
  (not args.spiral and args.noise) or\
  (args.spiral and (args.training_set or args.input)) or\
  (args.spiral and not args.grid_size) or\
  (args.input and not args.training_set and not args.slice) or\
  (args.training_set and args.slice):

   parser.print_help()
   sys.exit(0)

training_header = []
training_set = []
if args.training_set:
    data = utils.load(args.training_set)
    training_header = data[0]
    training_set = data[1:]

input_set = []
if args.input:
    data = utils.load(args.input)
    if not args.training_set:
        training_header = data[0]
    input_set = data[1:]

distance_function = None
if args.distance == 'euclidian':
    distance_function = cl.euclidian_dist
else:
    distance_function = cl.hamming_dist

if args.slice:
    num_picked_entries = floor(len(input_set) * args.slice / 100)
    num_entries = len(input_set)
    i = 0
    picked_indexes = set()
    while len(training_set) < num_picked_entries:
        if i >= num_entries:
            i = 1
        training_set.append(input_set[i])
        picked_indexes.add(i)
        i += 2

    for i in range(num_entries - 1, -1, -1):
        if i in picked_indexes:
            del input_set[i]

if args.ignore:
    args.ignore.sort(reverse=True)
    for entry in training_set:
        for index in args.ignore:
            del entry[index]

    for entry in input_set:
        for index in args.ignore:
            del entry[index]

    for index in args.ignore:
        if index < args.category:
            args.category -= 1
        del training_header[index]

target_file = args.output

output = []
if not args.spiral:
    hits = 0
    fails = 0
    output.append(training_header)
    for entry in input_set:
        prepared_entry = utils.without_column(entry, args.category)
        class_value = cl.kNN(training_set, prepared_entry, distance_function, \
                             args.category, args.k)
        output.append(prepared_entry + [class_value])
        if entry[args.category] != None:
            if entry[args.category] == class_value:
                hits += 1
            else:
                fails += 1

    utils.save(target_file, output)
    if args.verbose and hits + fails > 0:
        print("Hits: %i" % (hits))
        print("Fails: %i" % (fails))
        print("Precision: %f%%" % (100 * hits / (hits + fails)))

    if args.plot and len(training_set[0]) == 3:
        import plotter as pl

        allcolors = [color for color in cnames]
        categories = list(set([x[args.category] for x in training_set]))

        pl.plot(
            [x[:2] for x in output[1:]],
            [x[2] for x in output[1:]],
            [x[:2] for x in training_set],
            [x[2] for x in training_set],
            utils.without_column(training_header, args.category),
            {categories[i]: allcolors[((i + 1) * 666) % len(allcolors)] for i in range(len(categories))}
        )
else:
    neighborhood = 2 * args.grid_size + 50
    if args.spiral == "single":
        spiral = single_spiral(args.grid_size, args.noise)
        size = args.grid_size
        spiral_points = [(spiral[i][0] + size, spiral[i][1] + size, 0) \
                         for i in range(len(spiral)) if not (i % 2)] + \
                        [(spiral[i][0] + size, spiral[i][1] + size, 1) \
                         for i in range(len(spiral)) if (i % 2)]
    else:
        spirals = double_spiral(args.grid_size, args.noise)
        spiral_points = [(s[0] + args.grid_size, s[1] + args.grid_size, 0) for s in spirals[0]]\
                      + [(s[0] + args.grid_size, s[1] + args.grid_size, 1) for s in spirals[1]]

    red_points = []
    blue_points = []
    for x in range(neighborhood):
        for y in range(neighborhood):
            entry = (x, y)
            class_value = cl.kNN(spiral_points, entry, distance_function, \
                                 2, args.k)
            if class_value == 0:
                red_points.append(entry)
            else:
                blue_points.append(entry)

    first = [(255, 0, 0) for point in red_points]
    second = [(0, 0, 255) for point in blue_points]

    if args.save_image:
        import imagesaver
        imagesaver.save(red_points + blue_points, first + second, neighborhood, args.output)

    if args.plot:
        import plotter as pl

        output = red_points + blue_points
        result = first + second

        pl.plot(
            output,
            result,
            spiral_points[:1],
            [(255, 0, 0) for point in spiral_points if point[2] == 0] + \
            [(0, 0, 255) for point in spiral_points if point[2] == 1],
            ['x', 'y'],
            {(255, 0, 0) : 'red', (0, 0, 255) : 'blue'}
        )<|endoftext|>


<commit_before>import os
from distutils.version import LooseVersion

import pytest
import xarray as xr

SAMPLE_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "sample-data")
TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, "regular_ll_sfc.grib")

@pytest.mark.skipif(
    LooseVersion(xr.__version__) < "0.18",
    reason="xarray new backend interface available for xarray version >= 0.18",
)
def test_plugin():
    engines = xr.backends.list_engines()
    cfgrib_entrypoint = engines["cfgrib"]
    assert cfgrib_entrypoint.__module__ == "cfgrib.xarray_entrypoint"

def test_xr_open_dataset():
    expected = {
        "latitude": 37,
        "longitude": 72,
    }

    ds = xr.open_dataset(TEST_DATA, engine="cfgrib")
    assert ds.dims == expected
    assert list(ds.data_vars) == ["skt"]

@pytest.mark.skipif(
    LooseVersion(xr.__version__) < "0.18",
    reason="xarray new backend interface available for xarray version >= 0.18",
)
def test_read():
    expected = {
        "latitude": 37,
        "longitude": 72,
    }
    import cfgrib.xarray_entrypoint

    opener = cfgrib.xarray_entrypoint.CfgribfBackendEntrypoint()
    ds = opener.open_dataset(TEST_DATA)
    assert ds.dims == expected
    assert list(ds.data_vars) == ["skt"]
<commit_msg>fix condition in tests<commit_after>import os
from distutils.version import LooseVersion

import pytest
try:
    import xarray as xr
    has_xarray = True
except:
    has_xarray = False

SAMPLE_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "sample-data")
TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, "regular_ll_sfc.grib")

@pytest.mark.skipif(
    not (has_xarray and (LooseVersion(xr.__version__) >= "0.18")),
    reason="required xarray >= 0.18",
)
def test_plugin():
    engines = xr.backends.list_engines()
    cfgrib_entrypoint = engines["cfgrib"]
    assert cfgrib_entrypoint.__module__ == "cfgrib.xarray_entrypoint"

@pytest.mark.skipif(
    not has_xarray,
    reason="required xarray",
)
def test_xr_open_dataset():
    expected = {
        "latitude": 37,
        "longitude": 72,
    }

    ds = xr.open_dataset(TEST_DATA, engine="cfgrib")
    assert ds.dims == expected
    assert list(ds.data_vars) == ["skt"]

@pytest.mark.skipif(
    not (has_xarray and (LooseVersion(xr.__version__) >= "0.18")),
    reason="required xarray >= 0.18",
)
def test_read():
    expected = {
        "latitude": 37,
        "longitude": 72,
    }
    import cfgrib.xarray_entrypoint

    opener = cfgrib.xarray_entrypoint.CfgribfBackendEntrypoint()
    ds = opener.open_dataset(TEST_DATA)
    assert ds.dims == expected
    assert list(ds.data_vars) == ["skt"]
<|endoftext|>


<commit_before>#!/usr/bin/python3

import argparse
import utils
import core

parser = argparse.ArgumentParser(description="Classify some data")

mtlxcg = parser.add_mutually_exclusive_group()
mtlxcg.add_argument("-p", "--plot", action="store_true", help="Plot the classified patterns (only works with 2D data)")
mtlxcg.add_argument("-s", "--show", action="store_true", help="Plot the classified patterns (only works with 2D data)")
parser.add_argument("-v", "--verbose", action="store_true", help="Print additional statistics")

parser.add_argument("algorithm", choices=["IB1", "IB2"], help="Classification algorithm")

dataparsers = parser.add_subparsers(help="Specify type of data to be classified", dest="data_type")
dataparsers.required = True

dataset = dataparsers.add_parser("dataset", help="Classify a given dataset")
dataset.add_argument("-i", "--input", type=str, required=True, metavar="FILE", help="Filename of the dataset")
dataset.add_argument("-c", "--category", type=int, required=True, metavar="INDEX", help="Column of the class")
dataset.add_argument("-d", "--data", type=str, metavar="FILE", help="Filename of the data to be classified")

dtstmtx = dataset.add_mutually_exclusive_group()
dtstmtx.add_argument("-e", "--test_set", type=str, metavar="FILE", help="Filename of the test set")
dtstmtx.add_argument("-s", "--split", type=float, metavar="PERCENTAGE", help="Automatically pick the given approximate percentage of the input dataset to use as the test set")

dataset.add_argument("-o", "--output", type=str, default="out", metavar="FILE", help="Save classified data as <FILE>.csv")
dataset.add_argument("-I", "--ignore", nargs="+", type=int, metavar="INDEX", help="Columns to be ignored")

#image = dataparsers.add_parser("image", help="Classify a given image")

spiral = dataparsers.add_parser("spiral", help="Classify a given spiral")
spiral.add_argument("-t", "--type", type=str, choices=["single", "double"], default="single", help="Type of spiral")
spiral.add_argument("-g", "--grid_size", type=int, default=50, help="Grid size to generate spiral (default 50)")
spiral.add_argument("-n", "--noise", type=int, default=0, help="Noise for the spiral.")
spiral.add_argument("-o", "--output", type=str, default="out", metavar="FILE", help="Save classified data as <FILE>.png")

# parser.add_argument("-C", "--compare", nargs="+", choices=["l2norm", "quadratic_mahalanobis", "linear_mahalanobis"], help="Compare with other algorithms")

def process_dataset(args):
    training_header, training_set = utils.load_csv(args.input)
    del training_set[0]
    if len(training_set) == 0:
        dataset.error("Invalid input: file does not exist or is empty.")

    test_set = []
    data = []
    if args.split:
        test_set = core.split_data(data=training_set, percentage=args.split)
    elif args.test_set:
        test_header, test_set = utils.load_csv(args.test_set)
        if len(test_set) == 0:
            dataset.error("Invalid test set: file does not exist or is empty.")
    elif args.data:
        data_header, data = utils.load_csv(args.data)
    else:
        dataset.error("one of the arguments -d/--data -e/--test_set -s/--split is required")

    if args.ignore:
        utils.ignore_columns(training_set, args.ignore)
        utils.ignore_columns(test_set, args.ignore)
        utils.ignore_columns(data, args.ignore)
        for index in args.ignore:
            if index < args.category:
                args.category -= 1

    return (training_header, training_set, test_set, data)

if __name__ == "__main__":
    args = parser.parse_args()
    if args.data_type == "dataset":
        header, training_set, test_set, data = process_dataset(args)
        params = {
            "algorithm": args.algorithm,
        }
        # output = core.magic_function_to_do_all(params, training_set, test_set, data, args.category)
    else:
        print("spiral")
<commit_msg>removed trash line<commit_after>#!/usr/bin/python3

import argparse
import utils
import core

parser = argparse.ArgumentParser(description="Classify some data")

mtlxcg = parser.add_mutually_exclusive_group()
mtlxcg.add_argument("-p", "--plot", action="store_true", help="Plot the classified patterns (only works with 2D data)")
mtlxcg.add_argument("-s", "--show", action="store_true", help="Plot the classified patterns (only works with 2D data)")
parser.add_argument("-v", "--verbose", action="store_true", help="Print additional statistics")

parser.add_argument("algorithm", choices=["IB1", "IB2"], help="Classification algorithm")

dataparsers = parser.add_subparsers(help="Specify type of data to be classified", dest="data_type")
dataparsers.required = True

dataset = dataparsers.add_parser("dataset", help="Classify a given dataset")
dataset.add_argument("-i", "--input", type=str, required=True, metavar="FILE", help="Filename of the dataset")
dataset.add_argument("-c", "--category", type=int, required=True, metavar="INDEX", help="Column of the class")
dataset.add_argument("-d", "--data", type=str, metavar="FILE", help="Filename of the data to be classified")

dtstmtx = dataset.add_mutually_exclusive_group()
dtstmtx.add_argument("-e", "--test_set", type=str, metavar="FILE", help="Filename of the test set")
dtstmtx.add_argument("-s", "--split", type=float, metavar="PERCENTAGE", help="Automatically pick the given approximate percentage of the input dataset to use as the test set")

dataset.add_argument("-o", "--output", type=str, default="out", metavar="FILE", help="Save classified data as <FILE>.csv")
dataset.add_argument("-I", "--ignore", nargs="+", type=int, metavar="INDEX", help="Columns to be ignored")

#image = dataparsers.add_parser("image", help="Classify a given image")

spiral = dataparsers.add_parser("spiral", help="Classify a given spiral")
spiral.add_argument("-t", "--type", type=str, choices=["single", "double"], default="single", help="Type of spiral")
spiral.add_argument("-g", "--grid_size", type=int, default=50, help="Grid size to generate spiral (default 50)")
spiral.add_argument("-n", "--noise", type=int, default=0, help="Noise for the spiral.")
spiral.add_argument("-o", "--output", type=str, default="out", metavar="FILE", help="Save classified data as <FILE>.png")

# parser.add_argument("-C", "--compare", nargs="+", choices=["l2norm", "quadratic_mahalanobis", "linear_mahalanobis"], help="Compare with other algorithms")

def process_dataset(args):
    training_header, training_set = utils.load_csv(args.input)
    if len(training_set) == 0:
        dataset.error("Invalid input: file does not exist or is empty.")

    test_set = []
    data = []
    if args.split:
        test_set = core.split_data(data=training_set, percentage=args.split)
    elif args.test_set:
        test_header, test_set = utils.load_csv(args.test_set)
        if len(test_set) == 0:
            dataset.error("Invalid test set: file does not exist or is empty.")
    elif args.data:
        data_header, data = utils.load_csv(args.data)
    else:
        dataset.error("one of the arguments -d/--data -e/--test_set -s/--split is required")

    if args.ignore:
        utils.ignore_columns(training_set, args.ignore)
        utils.ignore_columns(test_set, args.ignore)
        utils.ignore_columns(data, args.ignore)
        for index in args.ignore:
            if index < args.category:
                args.category -= 1

    return (training_header, training_set, test_set, data)

if __name__ == "__main__":
    args = parser.parse_args()
    if args.data_type == "dataset":
        header, training_set, test_set, data = process_dataset(args)
        params = {
            "algorithm": args.algorithm,
        }
        # output = core.magic_function_to_do_all(params, training_set, test_set, data, args.category)
    else:
        print("spiral")
<|endoftext|>


<commit_before>import os
import sys
lib_path = os.path.abspath('./')
sys.path.insert(0, lib_path)

try:
    import unittest2 as unittest
except ImportError:
    import unittest
import mock

import tempfile

import makerbot_driver
import warnings

class TOMReading(unittest.TestCase):
    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.p.state = makerbot_driver.Gcode.LegacyGcodeStates()
        self.p.state.values['build_name'] = 'test'
        self.p.state.profile = makerbot_driver.Profile('TOMStepstruderSingle')
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=True) as f:
            path = f.name
        self.s3g.writer = makerbot_driver.Writer.FileWriter(open(path, 'wb'))
        self.p.s3g = self.s3g

    def tearDown(self):
        self.p = None
        self.s3g = None

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode'
        )
        execute_file(the_file, self.p)

class SingleHeadReading(unittest.TestCase):

    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.s = makerbot_driver.Gcode.GcodeStates()
        self.s.values['build_name'] = 'test'
        self.profile = makerbot_driver.Profile('ReplicatorSingle')
        self.s.profile = self.profile
        self.p.state = self.s
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=False) as input_file:
            pass
        input_path = input_file.name
        os.unlink(input_path)
        self.writer = makerbot_driver.Writer.FileWriter(open(input_path, 'wb'))
        self.s3g.writer = self.writer
        self.p.s3g = self.s3g

    def tearDown(self):
        self.profile = None
        self.s = None
        self.writer = None
        self.s3g = None
        self.p = None

    def test_single_head_slicer(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'slic3r_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'SlicerProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_20mm_box(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_snake(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_snake.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode'
        )
        execute_file(the_file, self.p)

class DualHeadReading(unittest.TestCase):

    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.s = makerbot_driver.Gcode.GcodeStates()
        self.s.values['build_name'] = 'test'
        self.profile = makerbot_driver.Profile('ReplicatorDual')
        self.s.profile = self.profile
        self.p.state = self.s
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=False) as input_file:
            pass
        input_path = input_file.name
        os.unlink(input_path)
        self.writer = makerbot_driver.Writer.FileWriter(open(input_path, 'wb'))
        self.s3g.writer = self.writer
        self.p.s3g = self.s3g

    def tearDown(self):
        self.profile = None
        self.s = None
        self.writer = None
        self.s3g = None
        self.p = None

    def test_dual_head_skeinforge_hilbert_cube(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_dual_extrusion_hilbert_cube.gcode')
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        the_file = process_file_with_pro(
            the_file, 'CoordinateRemovalProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_20mm_box(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_snake(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_snake.gcode')
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode',
        )
        execute_file(the_file, self.p)

def process_file_with_pro(the_file, pro):
    factory = makerbot_driver.GcodeProcessors.ProcessorFactory()
    pro = factory.create_processor_from_name(pro)
    f = open(the_file)
    lines = list(f)
    output = pro.process_gcode(lines)
    with tempfile.NamedTemporaryFile(delete=False, suffix='.gcode') as f:
        processed_file_path = f.name
        for line in output:
            f.write(line)
    return processed_file_path

def execute_file(the_file, parser):
    ga = makerbot_driver.GcodeAssembler(parser.state.profile)
    if "Thing-O-Matic" in parser.state.profile.values['machinenames']:
        start, end, variables = ga.assemble_recipe(
            begin_print='tom_begin',
            homing='tom_homing',
            start_position='tom_start_position',
            end_start_sequence='tom_end_start_sequence',
            end_position='tom_end_position',
            end_print='tom_end',
        )
    else:
        start, end, variables = ga.assemble_recipe()
    start_gcode = ga.assemble_start_sequence(start)
    end_gcode = ga.assemble_end_sequence(end)
    parser.environment.update(variables)
    for line in start_gcode:
        parser.execute_line(line)
    with open(the_file) as f:
        for line in f:
            parser.execute_line(line)
    for line in end_gcode:
        parser.execute_line(line)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixing some broken tests.<commit_after>import os
import sys
lib_path = os.path.abspath('./')
sys.path.insert(0, lib_path)

try:
    import unittest2 as unittest
except ImportError:
    import unittest
import mock

import tempfile

import makerbot_driver
import warnings

class TOMReading(unittest.TestCase):
    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.p.state = makerbot_driver.Gcode.LegacyGcodeStates()
        self.p.state.values['build_name'] = 'test'
        self.p.state.profile = makerbot_driver.Profile('TOMStepstruderSingle')
        start_pos = self.p.state.profile.values['print_start_sequence']['start_position']
        start_position = {
            'START_X' : start_pos['start_x'],
            'START_Y' : start_pos['start_y'],
            'START_Z' : start_pos['start_z']
        }
        self.p.environment.update(start_position)
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=True) as f:
            path = f.name
        self.s3g.writer = makerbot_driver.Writer.FileWriter(open(path, 'wb'))
        self.p.s3g = self.s3g

    def tearDown(self):
        self.p = None
        self.s3g = None

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode'
        )
        execute_file(the_file, self.p)

class SingleHeadReading(unittest.TestCase):

    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.s = makerbot_driver.Gcode.GcodeStates()
        self.s.values['build_name'] = 'test'
        self.profile = makerbot_driver.Profile('ReplicatorSingle')
        start_pos = self.profile.values['print_start_sequence']['start_position']
        start_position = {
            'START_X' : start_pos['start_x'],
            'START_Y' : start_pos['start_y'],
            'START_Z' : start_pos['start_z']
        }
        self.p.environment.update(start_position)
        self.s.profile = self.profile
        self.p.state = self.s
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=False) as input_file:
            pass
        input_path = input_file.name
        os.unlink(input_path)
        self.writer = makerbot_driver.Writer.FileWriter(open(input_path, 'wb'))
        self.s3g.writer = self.writer
        self.p.s3g = self.s3g

    def tearDown(self):
        self.profile = None
        self.s = None
        self.writer = None
        self.s3g = None
        self.p = None

    def test_single_head_slicer(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'slic3r_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'SlicerProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_20mm_box(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_snake(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_snake.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode'
        )
        execute_file(the_file, self.p)

class DualHeadReading(unittest.TestCase):

    def setUp(self):
        self.p = makerbot_driver.Gcode.GcodeParser()
        self.s = makerbot_driver.Gcode.GcodeStates()
        self.s.values['build_name'] = 'test'
        self.profile = makerbot_driver.Profile('ReplicatorDual')
        start_pos = self.profile.values['print_start_sequence']['start_position']
        start_position = {
            'START_X' : start_pos['start_x'],
            'START_Y' : start_pos['start_y'],
            'START_Z' : start_pos['start_z']
        }
        self.p.environment.update(start_position)
        self.s.profile = self.profile
        self.p.state = self.s
        self.s3g = makerbot_driver.s3g()
        with tempfile.NamedTemporaryFile(suffix='.gcode', delete=False) as input_file:
            pass
        input_path = input_file.name
        os.unlink(input_path)
        self.writer = makerbot_driver.Writer.FileWriter(open(input_path, 'wb'))
        self.s3g.writer = self.writer
        self.p.s3g = self.s3g

    def tearDown(self):
        self.profile = None
        self.s = None
        self.writer = None
        self.s3g = None
        self.p = None

    def test_dual_head_skeinforge_hilbert_cube(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_dual_extrusion_hilbert_cube.gcode')
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        the_file = process_file_with_pro(
            the_file, 'CoordinateRemovalProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_20mm_box(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_20mm_box.gcode'
        )
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_skeinforge_single_snake(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'skeinforge_single_extrusion_snake.gcode')
        the_file = process_file_with_pro(the_file, 'Skeinforge50Processor')
        the_file = process_file_with_pro(the_file, 'SetTemperatureProcessor')
        execute_file(the_file, self.p)

    def test_single_head_miracle_grue(self):
        the_file = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            '..',
            'doc',
            'gcode_samples',
            'miracle_grue_single_extrusion_20_mm_box.gcode',
        )
        execute_file(the_file, self.p)

def process_file_with_pro(the_file, pro):
    factory = makerbot_driver.GcodeProcessors.ProcessorFactory()
    pro = factory.create_processor_from_name(pro)
    f = open(the_file)
    lines = list(f)
    output = pro.process_gcode(lines)
    with tempfile.NamedTemporaryFile(delete=False, suffix='.gcode') as f:
        processed_file_path = f.name
        for line in output:
            f.write(line)
    return processed_file_path

def execute_file(the_file, parser):
    ga = makerbot_driver.GcodeAssembler(parser.state.profile)
    if "Thing-O-Matic" in parser.state.profile.values['machinenames']:
        start, end, variables = ga.assemble_recipe(
            begin_print='tom_begin',
            homing='tom_homing',
            start_position='tom_start_position',
            end_start_sequence='tom_end_start_sequence',
            end_position='tom_end_position',
            end_print='tom_end',
        )
    else:
        start, end, variables = ga.assemble_recipe()
    start_gcode = ga.assemble_start_sequence(start)
    end_gcode = ga.assemble_end_sequence(end)
    parser.environment.update(variables)
    for line in start_gcode:
        parser.execute_line(line)
    with open(the_file) as f:
        for line in f:
            parser.execute_line(line)
    for line in end_gcode:
        parser.execute_line(line)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import numpy

from chainer.functions.normalization._standardize import _standardize
from chainer import testing
from chainer import utils

@testing.parameterize(*testing.product([
    [
        {'ch_out': 1},
        {'ch_out': 5},
    ],
    [
        {'size': 10},
        {'size': 20},
    ],
    [
        {'dtype': numpy.float64},
        {'dtype': numpy.float32},
        {'dtype': numpy.float16},
    ],
    [
        # same (str): flag whether input elems are same values.
        #   'no'   : all elems are randamly-chosen,
        #   'equal': all elems are equal,
        #   'near' : all elems are (randomly-chosen small values
        #            + same value).
        {'eps': 1e-5, 'same': 'no'},
        {'eps': 1e-1, 'same': 'no'},
        {'eps': 1e-1, 'same': 'equal'},
        {'eps': 1e-1, 'same': 'near'},
    ],
]))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    testing.product({
        'use_cuda': [False],
        'use_ideep': ['never', 'always'],
    })
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'use_cudnn': ['never', 'always'],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestStandardize(testing.FunctionTestCase):

    def setUp(self):
        if self.dtype == numpy.float16:
            self.check_forward_options.update({'atol': 2e-3, 'rtol': 2e-3})
            self.check_backward_options.update({'atol': 5e-3, 'rtol': 1e-2})
            self.check_double_backward_options.update({
                'atol': 5e-3, 'rtol': 1e-2})
        if self.same in ('equal', 'near'):
            self.check_backward_options.update({
                'atol': 1e-2, 'rtol': 1e-2, 'eps': 1e-4})
            self.skip_double_backward_test = True

    def generate_inputs(self):
        shape = self.ch_out, self.size
        if self.same in ('equal', 'near'):
            # Make self.x have same values
            x = numpy.ones(shape, self.dtype)
            x *= numpy.random.uniform(-1, 1)
            if self.same == 'near':
                # Make self.x have slightly different values
                zero_scale = 10. ** numpy.random.randint(-16, -3)
                x *= 1. + numpy.random.uniform(-zero_scale, zero_scale, shape)
        else:
            x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return _standardize(x, self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        mu = numpy.mean(x, axis=1, keepdims=True)
        x_mu = x - mu
        var = numpy.mean(numpy.square(x_mu), axis=1, keepdims=True)
        std = numpy.sqrt(var, dtype=x.dtype) + x.dtype.type(self.eps)
        return utils.force_array(x_mu / std, dtype=self.dtype),

testing.run_module(__name__, __file__)
<commit_msg>Fix tolerance<commit_after>import numpy

from chainer.functions.normalization._standardize import _standardize
from chainer import testing
from chainer import utils

@testing.parameterize(*testing.product([
    [
        {'ch_out': 1},
        {'ch_out': 5},
    ],
    [
        {'size': 10},
        {'size': 20},
    ],
    [
        {'dtype': numpy.float64},
        {'dtype': numpy.float32},
        {'dtype': numpy.float16},
    ],
    [
        # same (str): flag whether input elems are same values.
        #   'no'   : all elems are randamly-chosen,
        #   'equal': all elems are equal,
        #   'near' : all elems are (randomly-chosen small values
        #            + same value).
        {'eps': 1e-5, 'same': 'no'},
        {'eps': 1e-1, 'same': 'no'},
        {'eps': 1e-1, 'same': 'equal'},
        {'eps': 1e-1, 'same': 'near'},
    ],
]))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    testing.product({
        'use_cuda': [False],
        'use_ideep': ['never', 'always'],
    })
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'use_cudnn': ['never', 'always'],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestStandardize(testing.FunctionTestCase):

    def setUp(self):
        if self.dtype == numpy.float16:
            self.check_forward_options.update({'atol': 5e-3, 'rtol': 2e-3})
            self.check_backward_options.update({'atol': 5e-3, 'rtol': 1e-2})
            self.check_double_backward_options.update({
                'atol': 5e-3, 'rtol': 1e-2})
        if self.same in ('equal', 'near'):
            self.check_backward_options.update({
                'atol': 1e-2, 'rtol': 1e-2, 'eps': 1e-4})
            self.skip_double_backward_test = True

    def generate_inputs(self):
        shape = self.ch_out, self.size
        if self.same in ('equal', 'near'):
            # Make self.x have same values
            x = numpy.ones(shape, self.dtype)
            x *= numpy.random.uniform(-1, 1)
            if self.same == 'near':
                # Make self.x have slightly different values
                zero_scale = 10. ** numpy.random.randint(-16, -3)
                x *= 1. + numpy.random.uniform(-zero_scale, zero_scale, shape)
        else:
            x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return _standardize(x, self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        mu = numpy.mean(x, axis=1, keepdims=True)
        x_mu = x - mu
        var = numpy.mean(numpy.square(x_mu), axis=1, keepdims=True)
        std = numpy.sqrt(var, dtype=x.dtype) + x.dtype.type(self.eps)
        return utils.force_array(x_mu / std, dtype=self.dtype),

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# _____         ______  ______        _________
# __  /_______ ____  /_ ___  /_ _____ ______  /
# _  __/_  __ `/__  __ \__  __ \_  _ \_  __  /
# / /_  / /_/ / _  /_/ /_  /_/ //  __// /_/ /
# \__/  \__,_/  /_.___/ /_.___/ \___/ \__,_/

import csv

from helpers import *
from packages import simplejson as json
from packages import xlwt

__all__ = ['Dataset', 'source']

__version__ = '0.0.3'
__build__ = '0x000003'
__author__ = 'Kenneth Reitz'
__license__ = 'MIT'
__copyright__ = 'Copyright 2010 Kenneth Reitz'

class Dataset(object):
	"""Amazing Tabular Dataset object. """

	def __init__(self, *args, **kwargs):
		self._data = None
		self._filename = None

		self._data = list(args)

		try:
		    self.headers = kwargs['headers']
		except KeyError, why:
			self.headers = None

	def __len__(self):
		return self.height

	def __getitem__(self, key):

		if is_string(key):
			if key in self.headers:
				pos = self.headers.index(key) # get 'key' index from each data
				return [row[pos] for row in self._data]
			else:
				raise KeyError
		else:
			return self._data[key]

	def __setitem__(self, key, value):
		self.validate(value)
		self._data[key] = value

	def __delitem__(self, key):
		del self._data[key]

	def __repr__(self):
		return '<dataset object>'

	def validate(self, row=None, safety=False):
		"""Assures size of every row in dataset is of proper proportions."""
		if row:
			is_valid = (len(row) == self.width) if self.width else True
		else:
			is_valid = all((len(x)== self.width for x in self._data))

		if is_valid:
			return True

		else:
			if not safety:
				raise InvalidDimensions
			return False

	def digest(self):
		"""Retruns digest information of dataset in human-readable format."""
		pass

	@property
	def height(self):
		"""Returns the height of the Dataset."""
		return len(self._data)

	@property
	def width(self):
		"""Returns the width of the Dataset."""

		try:
		    return len(self._data[0])
		except KeyError, why:
		    return 0

	@property
	def json(self):
		if self.headers:
			pass

	@property
	def yaml(self):
		# TODO: YAML Export
		pass

	@property
	def csv(self):
		# TODO: CSV Export
		pass

	@property
	def xls(self):
		# TODO: XLS Export
		pass

	def append(self, row, index=None):
		# todo: impliment index
		self.validate(row)
		self._data.append(row)

	def sort_by(self, key):
		"""SORTS datastet by given key"""
		# todo: accpept string if headers, or index nubmer
		pass

	def save(self):
		pass

		# note export format
		# open file, save the bitch

class InvalidDimensions(Exception):
	"Invalid size"

def source(io_string=None, filename=None):
	"""docstring for import"""
	#open by filename
	pass<commit_msg>Dataset Title Metatdata. Improved dataset repr.<commit_after># -*- coding: utf-8 -*-

# _____         ______  ______        _________
# __  /_______ ____  /_ ___  /_ _____ ______  /
# _  __/_  __ `/__  __ \__  __ \_  _ \_  __  /
# / /_  / /_/ / _  /_/ /_  /_/ //  __// /_/ /
# \__/  \__,_/  /_.___/ /_.___/ \___/ \__,_/

import csv
import os

from helpers import *
from packages import simplejson as json
from packages import xlwt
from packages import yaml

__all__ = ['Dataset', 'source']

__version__ = '0.0.3'
__build__ = '0x000003'
__author__ = 'Kenneth Reitz'
__license__ = 'MIT'
__copyright__ = 'Copyright 2010 Kenneth Reitz'

class Dataset(object):
	"""Amazing Tabular Dataset object. """

	def __init__(self, *args, **kwargs):
		self._data = None
		self._filename = None

		self._data = list(args)

		try:
		    self.headers = kwargs['headers']
		except KeyError, why:
			self.headers = None

		try:
		    self.title = kwargs['title']
		except KeyError, why:
			self.title = None

	def __len__(self):
		return self.height

	def __getitem__(self, key):

		if is_string(key):
			if key in self.headers:
				pos = self.headers.index(key) # get 'key' index from each data
				return [row[pos] for row in self._data]
			else:
				raise KeyError
		else:
			return self._data[key]

	def __setitem__(self, key, value):
		self.validate(value)
		self._data[key] = value

	def __delitem__(self, key):
		del self._data[key]

	def __repr__(self):
		if self.title:
			return '<%s dataset>' % (self.title.lower())
		else:
			return '<dataset object>'

	def validate(self, row=None, safety=False):
		"""Assures size of every row in dataset is of proper proportions."""
		if row:
			is_valid = (len(row) == self.width) if self.width else True
		else:
			is_valid = all((len(x)== self.width for x in self._data))

		if is_valid:
			return True

		else:
			if not safety:
				raise InvalidDimensions
			return False

	def digest(self):
		"""Retruns digest information of dataset in human-readable format."""
		'Height: Width: '

	@property
	def height(self):
		"""Returns the height of the Dataset."""
		return len(self._data)

	@property
	def width(self):
		"""Returns the width of the Dataset."""

		try:
		    return len(self._data[0])
		except KeyError, why:
		    return 0

	@property
	def json(self):
		if self.headers:
			pass

	@property
	def yaml(self):
		# TODO: YAML Export
		pass

	@property
	def csv(self):
		# TODO: CSV Export
		pass

	@property
	def xls(self):
		# TODO: XLS Export
		pass

	def append(self, row, index=None):
		# todo: impliment index
		self.validate(row)
		self._data.append(row)

	def sort_by(self, key):
		"""SORTS datastet by given key"""
		# todo: accpept string if headers, or index nubmer
		pass

	def save(self):
		pass

		# note export format
		# open file, save the bitch

class InvalidDimensions(Exception):
	"Invalid size"

def source(io_string=None, filename=None):
	"""docstring for import"""
	#open by filename
	pass<|endoftext|>


<commit_before>from ltk.constants import CONF_DIR, CONF_FN
import os
import shutil
import time

def create_config():
    """
    create config folder and file to initialize without auth
    """
    conf_path = os.path.join(os.getcwd(), CONF_DIR)
    try:
        os.mkdir(conf_path)
    except OSError:
        pass
    new_config_file_path = os.path.join(conf_path, CONF_FN)
    # find way to always get correct path to config file, regardless of current working directory
    path = os.path.join(os.getcwd(),'config')
    shutil.copyfile(path, new_config_file_path)

def create_txt_file(file_name, path=None):
    print("create_txt_file")
    if path:
        file_path = os.path.join(path, file_name)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    with open(file_path, 'w') as txt_file:
        txt_file.write('This is a sample text file. ')
        txt_file.close()
    return file_path

def delete_file(file_name, file_path=None):
    print("delete_file")
    if file_path:
        file_path = os.path.join(file_path, file_name)
        print(file_path)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    os.remove(file_path)

def create_directory(dir_path):
    print("creating directory: "+dir_path)
    try:
        if not os.path.exists(dir_path):
            os.mkdir(dir_path)
        else:
            if delete_directory(dir_path):
                os.mkdir(dir_path)
    except OSError:
        pass

def delete_directory(dir_path):
    print("deleting directory: "+dir_path)
    if os.path.exists(dir_path) and os.path.isdir(dir_path) and os.listdir(dir_path)==[]:
        os.rmdir(dir_path)
        return True
    else:
        return False

def append_file(file_name, path=None):
    if path:
        file_path = os.path.join(path, file_name)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    with open(file_path, 'a') as txt_file:
        txt_file.write('Appended text. ')
        print("appended text to file "+file_name)
        txt_file.close()

    return file_path

def poll_doc(action, doc_id):
    """polls Lingotek for the status of a document given an id
        :returns True if document is imported within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            return True
        time.sleep(5)
        time_passed += 1
    return False

def poll_rm(action, doc_id):
    """polls Lingotek for the status of a document given an id
        :returns True if document is deleted within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 404:
            return True
        time.sleep(1)
        time_passed += 1
    return False

def check_updated_ids(action, doc_ids):
    """polls Lingotek for the modification of multiple documents by id
        :returns True if documents are modified within 3 min, else False
    """
    orig_dates = {}
    for doc_id in doc_ids:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            orig_date = response.json()['properties']['modified_date']
            orig_dates[doc_id] = orig_date
            # orig_count = response.json()['entities'][1]['properties']['count']['character']
        else:
            print("Document id not found on Lingotek Cloud: "+str(doc_id))
            return False
    for doc_id in doc_ids:
        if not check_updated(action, doc_id, orig_dates[doc_id]):
            return False
    return True

def check_updated(action, doc_id, orig_date):
    """polls Lingotek for modification of a document given an id
        :returns True if document is modified within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            mod_date = response.json()['properties']['modified_date']
            # print("mod_date: "+str(mod_date))
            new_count = response.json()['entities'][1]['properties']['count']['character']
            # print("character count: "+str(new_count))
            # if (orig_count != new_count):
            if (mod_date != orig_date):
                return True
        else:
            print("Document id not found on Lingotek Cloud: "+str(doc_id))
            return False
        time.sleep(5)
        time_passed += 1
    return False

def cleanup():
    conf_path = os.path.join(os.getcwd(), CONF_DIR)
    try:
        shutil.rmtree(conf_path)
    except OSError:
        print ('OSError in cleanup')
<commit_msg>print out clearer message when deleting a directory fails<commit_after>from ltk.constants import CONF_DIR, CONF_FN
import os
import shutil
import time

def create_config():
    """
    create config folder and file to initialize without auth
    """
    conf_path = os.path.join(os.getcwd(), CONF_DIR)
    try:
        os.mkdir(conf_path)
    except OSError:
        pass
    new_config_file_path = os.path.join(conf_path, CONF_FN)
    # find way to always get correct path to config file, regardless of current working directory
    path = os.path.join(os.getcwd(),'config')
    shutil.copyfile(path, new_config_file_path)

def create_txt_file(file_name, path=None):
    print("create_txt_file")
    if path:
        file_path = os.path.join(path, file_name)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    with open(file_path, 'w') as txt_file:
        txt_file.write('This is a sample text file. ')
        txt_file.close()
    return file_path

def delete_file(file_name, file_path=None):
    print("delete_file")
    if file_path:
        file_path = os.path.join(file_path, file_name)
        print(file_path)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    os.remove(file_path)

def create_directory(dir_path):
    print("creating directory: "+dir_path)
    try:
        if not os.path.exists(dir_path):
            os.mkdir(dir_path)
        else:
            if delete_directory(dir_path):
                os.mkdir(dir_path)
    except OSError:
        pass

def delete_directory(dir_path):
    print("deleting directory: "+dir_path)
    if os.path.exists(dir_path) and os.path.isdir(dir_path) and os.listdir(dir_path)==[]:
        os.rmdir(dir_path)
        return True
    else:
        print("directory " + dir_path + " is not empty, and could not be deleted")
        return False

def append_file(file_name, path=None):
    if path:
        file_path = os.path.join(path, file_name)
    else:
        file_path = os.path.join(os.getcwd(), file_name)
    with open(file_path, 'a') as txt_file:
        txt_file.write('Appended text. ')
        print("appended text to file "+file_name)
        txt_file.close()

    return file_path

def poll_doc(action, doc_id):
    """polls Lingotek for the status of a document given an id
        :returns True if document is imported within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            return True
        time.sleep(5)
        time_passed += 1
    return False

def poll_rm(action, doc_id):
    """polls Lingotek for the status of a document given an id
        :returns True if document is deleted within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 404:
            return True
        time.sleep(1)
        time_passed += 1
    return False

def check_updated_ids(action, doc_ids):
    """polls Lingotek for the modification of multiple documents by id
        :returns True if documents are modified within 3 min, else False
    """
    orig_dates = {}
    for doc_id in doc_ids:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            orig_date = response.json()['properties']['modified_date']
            orig_dates[doc_id] = orig_date
            # orig_count = response.json()['entities'][1]['properties']['count']['character']
        else:
            print("Document id not found on Lingotek Cloud: "+str(doc_id))
            return False
    for doc_id in doc_ids:
        if not check_updated(action, doc_id, orig_dates[doc_id]):
            return False
    return True

def check_updated(action, doc_id, orig_date):
    """polls Lingotek for modification of a document given an id
        :returns True if document is modified within 3 min, else False
    """
    time_passed = 0
    while time_passed < 180:
        response = action.api.get_document(doc_id)
        if response.status_code == 200:
            mod_date = response.json()['properties']['modified_date']
            # print("mod_date: "+str(mod_date))
            new_count = response.json()['entities'][1]['properties']['count']['character']
            # print("character count: "+str(new_count))
            # if (orig_count != new_count):
            if (mod_date != orig_date):
                return True
        else:
            print("Document id not found on Lingotek Cloud: "+str(doc_id))
            return False
        time.sleep(5)
        time_passed += 1
    return False

def cleanup():
    conf_path = os.path.join(os.getcwd(), CONF_DIR)
    try:
        shutil.rmtree(conf_path)
    except OSError:
        print ('OSError in cleanup')
<|endoftext|>