<commit_before>import utils

subscriptions = {}
SKIPLOG = set()
qtile = None

def init(q):
    global qtile
    qtile = q

def clear():
    subscriptions.clear()

class Subscribe:
    def __init__(self):
        hooks = set([])
        for i in dir(self):
            if not i.startswith("_"):
                hooks.add(i)
        self.hooks = hooks

    def _subscribe(self, event, func):
        lst = subscriptions.setdefault(event, [])
        if not func in lst:
            lst.append(func)

    def startup(self, func):
        """
            Called when Qtile has initialized
        """
        return self._subscribe("startup", func)

    def setgroup(self, func):
        """
            Called when group is changed.
        """
        return self._subscribe("setgroup", func)

    def addgroup(self, func):
        """
            Called when group is added.
        """
        return self._subscribe("addgroup", func)

    def delgroup(self, func):
        """
            Called when group is deleted.
        """
        return self._subscribe("delgroup", func)

    def changegroup(self, func):
        """
            Called whenever a group change occurs.
        """
        return self._subscribe("changegroup", func)

    def focus_change(self, func):
        """
            Called when focus is changed.
        """
        return self._subscribe("focus_change", func)

    def float_change(self, func):
        """
            Called when a change in float state is made
        """
        return self._subscribe("float_change", func)

    def group_window_add(self, func):
        """
            Called when a new window is added to a group.
        """
        return self._subscribe("group_window_add", func)

    def window_name_change(self, func):
        """
            Called whenever a windows name changes.
        """
        return self._subscribe("window_name_change", func)

    def client_new(self, func):
        """
            Called before Qtile starts managing a new client. Use this hook to
            declare windows static, or add them to a group on startup. This
            hook is not called for internal windows.

            - arguments: window.Window object

            ## Example:

                def func(c):
                    if c.name == "xterm":
                        c.togroup("a")
                    elif c.name == "dzen":
                        c.static(0)
                libqtile.hook.subscribe.client_new(func)
        """
        return self._subscribe("client_new", func)

    def client_managed(self, func):
        """
            Called after Qtile starts managing a new client. That is, after a
            window is assigned to a group, or when a window is made static.
            This hook is not called for internal windows.

            - arguments: window.Window object
        """
        return self._subscribe("client_managed", func)

    def client_killed(self, func):
        """
            Called after a client has been unmanaged.

            - arguments: window.Window object of the killed window.
        """
        return self._subscribe("client_killed", func)

    def client_state_changed(self, func):
        """
            Called whenever client state changes.
        """
        return self._subscribe("client_state_changed", func)

    def client_type_changed(self, func):
        """
            Called whenever window type changes.
        """
        return self._subscribe("client_type_changed", func)

    def client_focus(self, func):
        """
            Called whenver focus changes.

            - arguments: window.Window object of the new focus.
        """
        return self._subscribe("client_focus", func)

    def client_mouse_enter(self, func):
        """
            Called when the mouse enters a client.
        """
        return self._subscribe("client_mouse_enter", func)

    def client_name_updated(self, func):
        """
            Called when the client name changes.
        """
        return self._subscribe("client_name_updated", func)

    def client_urgent_hint_changed(self, func):
        """
            Called when the client urgent hint changes.
        """
        return self._subscribe("client_urgent_hint_changed", func)

    def layout_change(self, func):
        """
            Called on layout change.
        """
        return self._subscribe("layout_change", func)

    def net_wm_icon_change(self, func):
        """
            Called on _NET_WM_ICON chance.
        """
        return self._subscribe("net_wm_icon_change", func)

    def screen_change(self, func):
        """
            Called when a screen is added or screen configuration is changed
            (via xrandr). The hook should take two arguments: the root qtile
            object and the ``xproto.randr.ScreenChangeNotify`` event. Common
            usage is simply to call ``qtile.cmd_restart()`` on each event (to
            restart qtile when there is a new monitor):

            ## Example:

                def restart_on_randr(qtile, ev):
                    qtile.cmd_restart()
        """
        return self._subscribe("screen_change", func)

subscribe = Subscribe()

class Unsubscribe(Subscribe):
    """
        This class mirrors subscribe, except the _subscribe member has been
        overridden to removed calls from hooks.
    """
    def _subscribe(self, event, func):
        lst = subscriptions.setdefault(event, [])
        try:
            lst.remove(func)
        except ValueError:
            raise utils.QtileError(
                "Tried to unsubscribe a hook that was not"
                " currently subscribed"
            )

unsubscribe = Unsubscribe()

def fire(event, *args, **kwargs):
    if event not in subscribe.hooks:
        raise utils.QtileError("Unknown event: %s" % event)
    if not event in SKIPLOG:
        qtile.log.info(
            "Internal event: %s(%s, %s)" %
            (event, args, kwargs)
        )
    for i in subscriptions.get(event, []):
        i(*args, **kwargs)
<commit_msg>Don't crash on exceptions from hooks<commit_after>import traceback
import utils

subscriptions = {}
SKIPLOG = set()
qtile = None

def init(q):
    global qtile
    qtile = q

def clear():
    subscriptions.clear()

class Subscribe:
    def __init__(self):
        hooks = set([])
        for i in dir(self):
            if not i.startswith("_"):
                hooks.add(i)
        self.hooks = hooks

    def _subscribe(self, event, func):
        lst = subscriptions.setdefault(event, [])
        if not func in lst:
            lst.append(func)

    def startup(self, func):
        """
            Called when Qtile has initialized
        """
        return self._subscribe("startup", func)

    def setgroup(self, func):
        """
            Called when group is changed.
        """
        return self._subscribe("setgroup", func)

    def addgroup(self, func):
        """
            Called when group is added.
        """
        return self._subscribe("addgroup", func)

    def delgroup(self, func):
        """
            Called when group is deleted.
        """
        return self._subscribe("delgroup", func)

    def changegroup(self, func):
        """
            Called whenever a group change occurs.
        """
        return self._subscribe("changegroup", func)

    def focus_change(self, func):
        """
            Called when focus is changed.
        """
        return self._subscribe("focus_change", func)

    def float_change(self, func):
        """
            Called when a change in float state is made
        """
        return self._subscribe("float_change", func)

    def group_window_add(self, func):
        """
            Called when a new window is added to a group.
        """
        return self._subscribe("group_window_add", func)

    def window_name_change(self, func):
        """
            Called whenever a windows name changes.
        """
        return self._subscribe("window_name_change", func)

    def client_new(self, func):
        """
            Called before Qtile starts managing a new client. Use this hook to
            declare windows static, or add them to a group on startup. This
            hook is not called for internal windows.

            - arguments: window.Window object

            ## Example:

                def func(c):
                    if c.name == "xterm":
                        c.togroup("a")
                    elif c.name == "dzen":
                        c.static(0)
                libqtile.hook.subscribe.client_new(func)
        """
        return self._subscribe("client_new", func)

    def client_managed(self, func):
        """
            Called after Qtile starts managing a new client. That is, after a
            window is assigned to a group, or when a window is made static.
            This hook is not called for internal windows.

            - arguments: window.Window object
        """
        return self._subscribe("client_managed", func)

    def client_killed(self, func):
        """
            Called after a client has been unmanaged.

            - arguments: window.Window object of the killed window.
        """
        return self._subscribe("client_killed", func)

    def client_state_changed(self, func):
        """
            Called whenever client state changes.
        """
        return self._subscribe("client_state_changed", func)

    def client_type_changed(self, func):
        """
            Called whenever window type changes.
        """
        return self._subscribe("client_type_changed", func)

    def client_focus(self, func):
        """
            Called whenver focus changes.

            - arguments: window.Window object of the new focus.
        """
        return self._subscribe("client_focus", func)

    def client_mouse_enter(self, func):
        """
            Called when the mouse enters a client.
        """
        return self._subscribe("client_mouse_enter", func)

    def client_name_updated(self, func):
        """
            Called when the client name changes.
        """
        return self._subscribe("client_name_updated", func)

    def client_urgent_hint_changed(self, func):
        """
            Called when the client urgent hint changes.
        """
        return self._subscribe("client_urgent_hint_changed", func)

    def layout_change(self, func):
        """
            Called on layout change.
        """
        return self._subscribe("layout_change", func)

    def net_wm_icon_change(self, func):
        """
            Called on _NET_WM_ICON chance.
        """
        return self._subscribe("net_wm_icon_change", func)

    def screen_change(self, func):
        """
            Called when a screen is added or screen configuration is changed
            (via xrandr). The hook should take two arguments: the root qtile
            object and the ``xproto.randr.ScreenChangeNotify`` event. Common
            usage is simply to call ``qtile.cmd_restart()`` on each event (to
            restart qtile when there is a new monitor):

            ## Example:

                def restart_on_randr(qtile, ev):
                    qtile.cmd_restart()
        """
        return self._subscribe("screen_change", func)

subscribe = Subscribe()

class Unsubscribe(Subscribe):
    """
        This class mirrors subscribe, except the _subscribe member has been
        overridden to removed calls from hooks.
    """
    def _subscribe(self, event, func):
        lst = subscriptions.setdefault(event, [])
        try:
            lst.remove(func)
        except ValueError:
            raise utils.QtileError(
                "Tried to unsubscribe a hook that was not"
                " currently subscribed"
            )

unsubscribe = Unsubscribe()

def fire(event, *args, **kwargs):
    if event not in subscribe.hooks:
        raise utils.QtileError("Unknown event: %s" % event)
    if not event in SKIPLOG:
        qtile.log.info(
            "Internal event: %s(%s, %s)" %
            (event, args, kwargs)
        )
    for i in subscriptions.get(event, []):
        try:
            i(*args, **kwargs)
        except Exception as e:
            qtile.log.error("Error in hook %s:\n%s" % (
                event, traceback.format_exc()))
<|endoftext|>


<commit_before># Copyright (c) 2019, Sean Vig. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from typing import Dict, List, Optional, Tuple, Union  # noqa: F401

from libqtile.command_client import InteractiveCommandClient
from libqtile.command_graph import (
    CommandGraphCall,
    CommandGraphNode,
    SelectorType,
)
from libqtile.command_interface import CommandInterface

class LazyCall:
    def __init__(self, call: CommandGraphCall, args: Tuple, kwargs: Dict) -> None:
        """The lazily evaluated command graph call

        Parameters
        ----------
        call : CommandGraphCall
            The call that is made
        args : Tuple
            The args passed to the call when it is evaluated.
        kwargs : Dict
            The kwargs passed to the call when it is evaluated.
        """
        self._call = call
        self._args = args
        self._kwargs = kwargs

        self._layout = None  # type: Optional[str]
        self._when_floating = True

    @property
    def selectors(self) -> List[SelectorType]:
        """The selectors for the given call"""
        return self._call.selectors

    @property
    def name(self) -> str:
        """The name of the given call"""
        return self._call.name

    @property
    def args(self) -> Tuple:
        """The args to the given call"""
        return self._args

    @property
    def kwargs(self) -> Dict:
        """The kwargs to the given call"""
        return self._kwargs

    def when(self, layout: Optional[str] = None,
             when_floating: bool = True) -> 'LazyCall':
        """Filter call activation per layout or floating state

        Parameters
        ----------
        layout : str or None
            Restrict call to given layout name. If None, call for all layouts.
            If 'floating', call if, and only if the current window is floating,
            ``when_floating`` is ignored in this case.
        when_floating : bool
            Call if the current window is floating.
        """
        self._layout = layout
        self._when_floating = when_floating
        return self

    def check(self, q) -> bool:
        cur_win_floating = q.current_window and q.current_window.floating

        if self._layout == 'floating':  # ignore _when_floating
            return cur_win_floating

        if cur_win_floating and not self._when_floating:
            return False

        if self._layout and q.current_layout.name != self._layout:
            return False

        return True

class LazyCommandInterface(CommandInterface):
    """A lazy loading command object

    Allows all commands and items to be resolved at run time, and returns
    lazily evaluated commands.
    """

    def execute(self, call: CommandGraphCall, args: Tuple, kwargs: Dict) -> LazyCall:
        """Lazily evaluate the given call"""
        return LazyCall(call, args, kwargs)

    def has_command(self, node: CommandGraphNode, command: str) -> bool:
        """Lazily resolve the given command"""
        return True

    def has_item(self, node: CommandGraphNode, object_type: str, item: Union[str, int]) -> bool:
        """Lazily resolve the given item"""
        return True

lazy = InteractiveCommandClient(LazyCommandInterface())
<commit_msg>libqtile/lazy: support multiple layouts in LazyCall.when<commit_after># Copyright (c) 2019, Sean Vig. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from typing import Dict, Iterable, List, Optional, Set, Tuple, Union  # noqa: F401

from libqtile.command_client import InteractiveCommandClient
from libqtile.command_graph import (
    CommandGraphCall,
    CommandGraphNode,
    SelectorType,
)
from libqtile.command_interface import CommandInterface

class LazyCall:
    def __init__(self, call: CommandGraphCall, args: Tuple, kwargs: Dict) -> None:
        """The lazily evaluated command graph call

        Parameters
        ----------
        call : CommandGraphCall
            The call that is made
        args : Tuple
            The args passed to the call when it is evaluated.
        kwargs : Dict
            The kwargs passed to the call when it is evaluated.
        """
        self._call = call
        self._args = args
        self._kwargs = kwargs

        self._layouts = set()  # type: Set[str]
        self._when_floating = True

    @property
    def selectors(self) -> List[SelectorType]:
        """The selectors for the given call"""
        return self._call.selectors

    @property
    def name(self) -> str:
        """The name of the given call"""
        return self._call.name

    @property
    def args(self) -> Tuple:
        """The args to the given call"""
        return self._args

    @property
    def kwargs(self) -> Dict:
        """The kwargs to the given call"""
        return self._kwargs

    def when(self, layout: Optional[Union[Iterable[str], str]] = None,
             when_floating: bool = True) -> 'LazyCall':
        """Enable call only for given layout(s) and floating state

        Parameters
        ----------
        layout : str, Iterable[str], or None
            Restrict call to given layout names. If None, enable call
            for all layouts.
            Can be a single name, a string of comma-separated names, or any
            iterable yielding layout names.
        when_floating : bool
            Enable call when the current window is floating.
        """
        if layout is not None:
            if isinstance(layout, str):
                self._layouts = set(l.strip() for l in layout.split(","))
            else:
                self._layouts = set(layout)

        self._when_floating = when_floating
        return self

    def check(self, q) -> bool:
        cur_win_floating = q.current_window and q.current_window.floating

        if cur_win_floating and not self._when_floating:
            return False

        if self._layouts and q.current_layout.name not in self._layouts:
            return False

        return True

class LazyCommandInterface(CommandInterface):
    """A lazy loading command object

    Allows all commands and items to be resolved at run time, and returns
    lazily evaluated commands.
    """

    def execute(self, call: CommandGraphCall, args: Tuple, kwargs: Dict) -> LazyCall:
        """Lazily evaluate the given call"""
        return LazyCall(call, args, kwargs)

    def has_command(self, node: CommandGraphNode, command: str) -> bool:
        """Lazily resolve the given command"""
        return True

    def has_item(self, node: CommandGraphNode, object_type: str, item: Union[str, int]) -> bool:
        """Lazily resolve the given item"""
        return True

lazy = InteractiveCommandClient(LazyCommandInterface())
<|endoftext|>


<commit_before>"""
ParametersFrame class.

$Id$
"""
__version__='$Revision$'

from propertiesframe import PropertiesFrame
from Tkinter import Frame, TOP, YES, N,S,E,W,X
import topo.base.utils
from topo.base.utils import keys_sorted_by_value
import topo
import topo.base.parameter
import Pmw

# CEBHACKALERT: this file is still being reorganized; there are still
# temporary methods.

# CEBHACKALERT: doesn't work for TopoObject class.

# CEBHACKALERT: there used to be a 'reset_to_defaults' method, which
# didn't work. When Parameters can be set and then maintained between
# classes (e.g. for PatternGenerators), reset_to_defaults() should be
# re-implemented so that Parameters can be returned to default values
# for the current class.
# To maintain values of Parameters between classes, the object using
# the ParametersFrame will need to do something like pass
# ParametersFrame a dictionary of {parameter:value} pairs, which
# ParametersFrame should be able to use selectively (i.e. use if
# relevant to the current class but ignore otherwise).

class ParametersFrame(Frame):
    """
    Frame for all non-hidden Parameters of a TopoObject class.

    When asked to create wigets, ... makes a PropertiesFrame containing all the specified class' Parameters.
    """

    def __init__(self, parent=None, **config):
        """
        """
        Frame.__init__(self,parent,config)
        self.__properties_frame = PropertiesFrame(parent)
        self.__properties_frame.pack(side=TOP,expand=YES,fill=X)

        # CEBHACKALERT: these buttons should be stacked on the window or
        # something, but I don't know how to do that.

        # buttons for setting/(restoring) class Parameter values.
        #pattern_buttonBox = Pmw.ButtonBox(self,orient = 'horizontal',padx=0,pady=0)
        #pattern_buttonBox.add('Install as class defaults',
        #                      command=self.install_as_class_defaults)
        #pattern_buttonBox.add('Revert to class defaults',
        #                      command=self.revert_to_class_defaults)

        # CEBHACKALERT: need this to revert to defaults in the file...
        #pattern_buttonBox.add('Restore class defaults',
        #                      command=self.restore_class_defaults)

        #pattern_buttonBox.pack(side=TOP)

        self.topo_obj = None

        self.__help_balloon = Pmw.Balloon(parent)

        self.__widgets = {} # should it be none?
        self.__visible_parameters = {}

        # The dictionary of parameter_type:property_to_add pairs.
        self.__parameter_property = {
            topo.base.parameter.Constant:         self.__add_readonly_text_property,
            topo.base.parameter.Number:           self.__add_numeric_property,
            topo.base.parameter.Enumeration:      self.__add_enumeration_property,
            topo.base.parameter.PackageParameter: self.__add_package_property,
            topo.base.parameter.BooleanParameter: self.__add_boolean_property}

    def install_as_class_defaults(self):
        """
        """
        assert self.topo_obj!=None, "ParametersFrame must be associated with a TopoObj()."

        # go through, get parameters, set them on the classobj
        t = type(self.topo_obj)

        for (name,parameter) in self.__visible_parameters.items():
            w = self.__widgets[name][1]
            setattr(t,name,w.get_value())

    # CEBHACKALERT
    def revert_to_class_defaults(self):
        pass

    # CEBHACKALERT
    def restore_class_defaults(self):
        """
        """
        pass

    # CEBHACKALERT: rename to set_object_parameters, probably.
    def set_obj_params(self):
        """
        For all non-Constant Parameters of the currently set TopoObject(),
        set the values of the Parameters to those specified by the widgets.
        """
        assert self.topo_obj!=None, "ParametersFrame must be associated with a TopoObj()."

        parameters_to_modify = [ (name,parameter)
                                 for (name,parameter)
                                 in self.__visible_parameters.items()
                                 if not type(parameter)==topo.base.parameter.Constant]

        for (name,parameter) in parameters_to_modify:
            w = self.__widgets[name][1]  # [0] is label (Message), [1] is widget
            setattr(self.topo_obj,name,w.get_value())

    def create_widgets(self, topo_obj):
        """
        Create widgets for all non-hidden Parameters of topo_obj and add them
        to the screen.

        Each Parameter gets a suitable widget (e.g. a slider for a Number with
        soft_bounds). The default widget is a text box.

        parameters must be Parameter objects.

        Widgets are added in order of the Parameters' precedences.
        """
        self.topo_obj=topo_obj

        # wipe old labels and widgets from screen
        for (label,widget) in self.__widgets.values():
            label.grid_forget()
            widget.grid_forget()

        # find visible parameters for topo_obj
        self.__visible_parameters = dict([(parameter_name,parameter)
                                        for (parameter_name,parameter)
                                        in self.topo_obj.get_paramobj_dict().items()
                                        if not parameter.hidden])

        # create the widgets
        self.__widgets = {}
        for (parameter_name, parameter) in self.__visible_parameters.items():
            self.__add_property_for_parameter(parameter_name,parameter)

        # sort Parameters by precedence (oops actually reverse of precedence!)
        parameter_precedences = {}
        for name,parameter in self.__visible_parameters.items():
            parameter_precedences[name] = parameter.precedence
        sorted_parameter_names = keys_sorted_by_value(parameter_precedences)

        # add widgets to screen
        rows = range(len(sorted_parameter_names))
        for (row,parameter_name) in zip(rows,sorted_parameter_names):
            (label,widget) = self.__widgets[parameter_name]
            help_text = self.__visible_parameters[parameter_name].__doc__

            label.grid(row=row,
                       column=0,
                       padx=self.__properties_frame.padding,
                       pady=self.__properties_frame.padding,
                       sticky=E)

            widget.grid(row=row,
                        column=1,
                        padx=self.__properties_frame.padding,
                        pady=self.__properties_frame.padding,
                        sticky=N+S+W+E)

            self.__help_balloon.bind(label, help_text)

    def __add_property_for_parameter(self,parameter_name,parameter):
        """
        Find the correct property type for the given parameter.

        If the parameter's type is not listed in
        self.__parameter_property, the class hierarchy is traversed until
        a match is found. If no match is found, the Parameter just gets a
        textbox.
        """
        for c in topo.base.parameter.classlist(type(parameter))[::-1]:
            if self.__parameter_property.has_key(c):
                # find the right method...
                property_to_add = self.__parameter_property[c]
                # ...then call it
                property_to_add(parameter_name,parameter)
                return
        # no match: use text box
        self.__add_text_property(parameter_name,parameter)

    # CEBHACKALERT: don't need to pass parameter. See HACKALERT below
    # about DynamicNumber
    def __add_text_property(self,parameter_name,parameter):
        """
        Add a text property to the properties_frame.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
            parameter_name,
            value = getattr(self.topo_obj,parameter_name))

    def __add_readonly_text_property(self,parameter_name,parameter):
        """
        Add a readonly text property to the properties_frame.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
            parameter_name,
            value = getattr(self.topo_obj,parameter_name),
            readonly = True)

    def __add_numeric_property(self,parameter_name,parameter):
        """
        Add a numeric property to the properties_frame.

        If the parameter has distinct low and high softbounds, a tagged_slider
        property is added. Otherwise, just a textbox is added (with
        a translator for string to float).
        """
        try:
            low_bound,high_bound = parameter.get_soft_bounds()

            if low_bound==None or high_bound==None or low_bound==high_bound:
                raise AttributeError # i.e. there aren't really softbounds

            # CEBHACKALERT: revert to previous behaviour for DynamicNumber
            # until we figure out how to do it properly.
            if isinstance(parameter, topo.base.parameter.DynamicNumber):
                v = parameter.default
            else:
                v = getattr(self.topo_obj,parameter_name)

            self.__widgets[parameter_name] = self.__properties_frame.add_tagged_slider_property(
                parameter_name,
                v,
                min_value = str(low_bound),
                max_value = str(high_bound),
                width = 30,
                string_format = '%.6f',
                translator = topo.base.utils.eval_atof)

        except AttributeError:
            self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
                parameter_name,
                value = getattr(self.topo_obj,parameter_name),
                translator = topo.base.utils.eval_atof)

    def __add_enumeration_property(self,parameter_name,parameter):
        """
        Add an enumeration property to the properties_frame by representing
        it with a ComboBox.
        """
        # an Enumeration gets a ComboBox
        # CEBHACKALERT: only string Enumerations work at the moment.
        self.__widgets[parameter_name] = self.__properties_frame.add_combobox_property(
                    parameter_name,
                    value = getattr(self.topo_obj,parameter_name),
                    scrolledlist_items = parameter.available)

    def __add_package_property(self,parameter_name,parameter):
        """
        Add a package property to the properties_frame by representing it
        with a ComboBox.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_combobox_property(
                    parameter_name,
                    value = parameter.get_default_class_name(), #.getattr(self.topo_obj,parameter_name),
                    scrolledlist_items = parameter.range().keys(),
                    translator = parameter.get_from_key)

    def __add_boolean_property(self,parameter_name,parameter):
        """
        Add a boolean property to the properties_frame by representing it with a
        Checkbutton.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_checkbutton_property(
                    parameter_name,
                    value = getattr(self.topo_obj,parameter_name))

<commit_msg>Commented out some currently unused code.<commit_after>"""
ParametersFrame class.

$Id$
"""
__version__='$Revision$'

from propertiesframe import PropertiesFrame
from Tkinter import Frame, TOP, YES, N,S,E,W,X
import topo.base.utils
from topo.base.utils import keys_sorted_by_value
import topo
import topo.base.parameter
import Pmw

# CEBHACKALERT: this file is still being reorganized; there are still
# temporary methods.

# CEBHACKALERT: doesn't work for TopoObject class.

# CEBHACKALERT: there used to be a 'reset_to_defaults' method, which
# didn't work. When Parameters can be set and then maintained between
# classes (e.g. for PatternGenerators), reset_to_defaults() should be
# re-implemented so that Parameters can be returned to default values
# for the current class.
# To maintain values of Parameters between classes, the object using
# the ParametersFrame will need to do something like pass
# ParametersFrame a dictionary of {parameter:value} pairs, which
# ParametersFrame should be able to use selectively (i.e. use if
# relevant to the current class but ignore otherwise).

class ParametersFrame(Frame):
    """
    Frame for all non-hidden Parameters of a TopoObject class.

    When asked to create wigets, ... makes a PropertiesFrame containing all the specified class' Parameters.
    """

    def __init__(self, parent=None, **config):
        """
        """
        Frame.__init__(self,parent,config)
        self.__properties_frame = PropertiesFrame(parent)
        self.__properties_frame.pack(side=TOP,expand=YES,fill=X)

        # CEBHACKALERT: these buttons should be stacked on the window or
        # something, but I don't know how to do that.

        # buttons for setting/(restoring) class Parameter values.
        #pattern_buttonBox = Pmw.ButtonBox(self,orient = 'horizontal',padx=0,pady=0)
        #pattern_buttonBox.add('Install as class defaults',
        #                      command=self.install_as_class_defaults)
        #pattern_buttonBox.add('Revert to class defaults',
        #                      command=self.revert_to_class_defaults)

        # CEBHACKALERT: need this to revert to defaults in the file...
        #pattern_buttonBox.add('Restore class defaults',
        #                      command=self.restore_class_defaults)

        #pattern_buttonBox.pack(side=TOP)

        self.topo_obj = None

        self.__help_balloon = Pmw.Balloon(parent)

        self.__widgets = {} # should it be none?
        self.__visible_parameters = {}

        # The dictionary of parameter_type:property_to_add pairs.
        self.__parameter_property = {
            topo.base.parameter.Constant:         self.__add_readonly_text_property,
            topo.base.parameter.Number:           self.__add_numeric_property,
            topo.base.parameter.Enumeration:      self.__add_enumeration_property,
            topo.base.parameter.PackageParameter: self.__add_package_property,
            topo.base.parameter.BooleanParameter: self.__add_boolean_property}

##     def install_as_class_defaults(self):
##         """
##         """
##         assert self.topo_obj!=None, "ParametersFrame must be associated with a TopoObj()."

##         # go through, get parameters, set them on the classobj
##         t = type(self.topo_obj)

##         for (name,parameter) in self.__visible_parameters.items():
##             w = self.__widgets[name][1]
##             setattr(t,name,w.get_value())

##     # CEBHACKALERT
##     def revert_to_class_defaults(self):
##         pass

##     # CEBHACKALERT
##     def restore_class_defaults(self):
##         """
##         """
##         pass

    # CEBHACKALERT: rename to set_object_parameters, probably.
    def set_obj_params(self):
        """
        For all non-Constant Parameters of the currently set TopoObject(),
        set the values of the Parameters to those specified by the widgets.
        """
        assert self.topo_obj!=None, "ParametersFrame must be associated with a TopoObj()."

        parameters_to_modify = [ (name,parameter)
                                 for (name,parameter)
                                 in self.__visible_parameters.items()
                                 if not type(parameter)==topo.base.parameter.Constant]

        for (name,parameter) in parameters_to_modify:
            w = self.__widgets[name][1]  # [0] is label (Message), [1] is widget
            setattr(self.topo_obj,name,w.get_value())

    def create_widgets(self, topo_obj):
        """
        Create widgets for all non-hidden Parameters of topo_obj and add them
        to the screen.

        Each Parameter gets a suitable widget (e.g. a slider for a Number with
        soft_bounds). The default widget is a text box.

        parameters must be Parameter objects.

        Widgets are added in order of the Parameters' precedences.
        """
        self.topo_obj=topo_obj

        # wipe old labels and widgets from screen
        for (label,widget) in self.__widgets.values():
            label.grid_forget()
            widget.grid_forget()

        # find visible parameters for topo_obj
        self.__visible_parameters = dict([(parameter_name,parameter)
                                        for (parameter_name,parameter)
                                        in self.topo_obj.get_paramobj_dict().items()
                                        if not parameter.hidden])

        # create the widgets
        self.__widgets = {}
        for (parameter_name, parameter) in self.__visible_parameters.items():
            self.__add_property_for_parameter(parameter_name,parameter)

        # sort Parameters by precedence (oops actually reverse of precedence!)
        parameter_precedences = {}
        for name,parameter in self.__visible_parameters.items():
            parameter_precedences[name] = parameter.precedence
        sorted_parameter_names = keys_sorted_by_value(parameter_precedences)

        # add widgets to screen
        rows = range(len(sorted_parameter_names))
        for (row,parameter_name) in zip(rows,sorted_parameter_names):
            (label,widget) = self.__widgets[parameter_name]
            help_text = self.__visible_parameters[parameter_name].__doc__

            label.grid(row=row,
                       column=0,
                       padx=self.__properties_frame.padding,
                       pady=self.__properties_frame.padding,
                       sticky=E)

            widget.grid(row=row,
                        column=1,
                        padx=self.__properties_frame.padding,
                        pady=self.__properties_frame.padding,
                        sticky=N+S+W+E)

            self.__help_balloon.bind(label, help_text)

    def __add_property_for_parameter(self,parameter_name,parameter):
        """
        Find the correct property type for the given parameter.

        If the parameter's type is not listed in
        self.__parameter_property, the class hierarchy is traversed until
        a match is found. If no match is found, the Parameter just gets a
        textbox.
        """
        for c in topo.base.parameter.classlist(type(parameter))[::-1]:
            if self.__parameter_property.has_key(c):
                # find the right method...
                property_to_add = self.__parameter_property[c]
                # ...then call it
                property_to_add(parameter_name,parameter)
                return
        # no match: use text box
        self.__add_text_property(parameter_name,parameter)

    # CEBHACKALERT: don't need to pass parameter. See HACKALERT below
    # about DynamicNumber
    def __add_text_property(self,parameter_name,parameter):
        """
        Add a text property to the properties_frame.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
            parameter_name,
            value = getattr(self.topo_obj,parameter_name))

    def __add_readonly_text_property(self,parameter_name,parameter):
        """
        Add a readonly text property to the properties_frame.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
            parameter_name,
            value = getattr(self.topo_obj,parameter_name),
            readonly = True)

    def __add_numeric_property(self,parameter_name,parameter):
        """
        Add a numeric property to the properties_frame.

        If the parameter has distinct low and high softbounds, a tagged_slider
        property is added. Otherwise, just a textbox is added (with
        a translator for string to float).
        """
        try:
            low_bound,high_bound = parameter.get_soft_bounds()

            if low_bound==None or high_bound==None or low_bound==high_bound:
                raise AttributeError # i.e. there aren't really softbounds

            # CEBHACKALERT: revert to previous behaviour for DynamicNumber
            # until we figure out how to do it properly.
            if isinstance(parameter, topo.base.parameter.DynamicNumber):
                v = parameter.default
            else:
                v = getattr(self.topo_obj,parameter_name)

            self.__widgets[parameter_name] = self.__properties_frame.add_tagged_slider_property(
                parameter_name,
                v,
                min_value = str(low_bound),
                max_value = str(high_bound),
                width = 30,
                string_format = '%.6f',
                translator = topo.base.utils.eval_atof)

        except AttributeError:
            self.__widgets[parameter_name] = self.__properties_frame.add_text_property(
                parameter_name,
                value = getattr(self.topo_obj,parameter_name),
                translator = topo.base.utils.eval_atof)

    def __add_enumeration_property(self,parameter_name,parameter):
        """
        Add an enumeration property to the properties_frame by representing
        it with a ComboBox.
        """
        # an Enumeration gets a ComboBox
        # CEBHACKALERT: only string Enumerations work at the moment.
        self.__widgets[parameter_name] = self.__properties_frame.add_combobox_property(
                    parameter_name,
                    value = getattr(self.topo_obj,parameter_name),
                    scrolledlist_items = parameter.available)

    def __add_package_property(self,parameter_name,parameter):
        """
        Add a package property to the properties_frame by representing it
        with a ComboBox.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_combobox_property(
                    parameter_name,
                    value = parameter.get_default_class_name(), #.getattr(self.topo_obj,parameter_name),
                    scrolledlist_items = parameter.range().keys(),
                    translator = parameter.get_from_key)

    def __add_boolean_property(self,parameter_name,parameter):
        """
        Add a boolean property to the properties_frame by representing it with a
        Checkbutton.
        """
        self.__widgets[parameter_name] = self.__properties_frame.add_checkbutton_property(
                    parameter_name,
                    value = getattr(self.topo_obj,parameter_name))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Thomas Reerink
#
# Run example:
#  python generate-ec-earth-namelists-for-all-MIP-experiments.py
#
# Looping over all MIPs and within each MIP over all its MIP experiments.
# The experiment tier can be selected. For each selected experiment the
# namelists are created by calling the generate-ec-earth-namelists.sh script.
#

import sys
import os
from dreqPy import dreq
dq = dreq.loadDreq()

# Specify which tier experiments should be included:
experiment_tiers_included = 1
#ec_earth_mips = ['CMIP'] # for basic test
ec_earth_mips = ['CMIP', 'DCPP']
#ec_earth_mips = ['CMIP', 'AerChemMIP', 'C4MIP',          'DAMIP', 'DCPP',                              'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP',         'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] # All 18 EC-Earth MIPs
#ec_earth_mips = ['CMIP', 'AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] # All 23 CMIP6 MIPs
experiment_counter = 0

command_0 = 'rm -rf cmip6-output-control-files'
os.system(command_0)

# Loop over MIPs:
for mip in dq.coll['mip'].items:
  # Loop over experiments:
  for u in dq.inx.iref_by_sect[mip.uid].a['experiment']:
    ex = dq.inx.uid[u]
    if ex.label == 'esm-hist' or ex.label == 'esm-piControl':
     print 'Skipping this esm experiment ' + ex.label + ' because its CMIP6 data request fails so far.\n'
    else:
     if experiment_counter == 0:
       command = './generate-ec-earth-namelists.sh ' + mip.label + ' ' + ex.label + ' ' + str(ex.tier[0]) + ' 1'
     else:
       command = './generate-ec-earth-namelists.sh ' + mip.label + ' ' + ex.label + ' ' + str(ex.tier[0]) + ' 1 omit-setup'
     command_2 = 'rm -rf cmip6-output-control-files/' + mip.label + '/cmip6-experiment-*/file_def-compact'
     command_3 = 'rm -f  cmip6-output-control-files/' + mip.label + '/cmip6-experiment-*/cmip6-file_def_nemo.xml'
     command_4 = "sed -i -e 's/True\" field_ref=\"toce_pot\"/False\" field_ref=\"toce_pot\"/' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-' + mip.label + '-' + ex.label + '/file_def_nemo-opa.xml'
     command_5 = "sed -i -e '/sfdsi_2/d' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-' + mip.label + '-' + ex.label + '/file_def_nemo-opa.xml'
    #command_4 = "sed -i -e 's/True\" field_ref=\"toce_pot\"/False\" field_ref=\"toce_pot\"/' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-m=' + mip.label + '-e=' + ex.label + '-t=' + str(ex.tier[0]) + '-p=1/file_def_nemo-opa.xml'
    #command_5 = "sed -i -e '/sfdsi_2/d' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-m=' + mip.label + '-e=' + ex.label + '-t=' + str(ex.tier[0]) + '-p=1/file_def_nemo-opa.xml'
    #print print '{}'.format(command)
     if mip.label in ec_earth_mips:
       #if ex.tier[0] == experiment_tiers_included and mip.label in ec_earth_mips and ex.label == 'piControl':  # for basic test
        if ex.tier[0] == experiment_tiers_included and mip.label in ec_earth_mips:
           os.system(command)
           os.system(command_2)
           os.system(command_3)
          #os.system(command_4) # Just set the toce fields false again because we still face troubles with them
           os.system(command_5) # Delete the line with sfdsi_2 from the file_def_nemo-opa.xml files
           experiment_counter = experiment_counter + 1
        else:
           print ' Tier {:2} experiments are not included: Skipping: {}'.format(ex.tier[0], command)
     else:
        print ' EC-Earth3 does not participate in {:2}: Skipping: {}'.format(mip.label, command)

print ' There are {} experiments included. '.format(experiment_counter)
<commit_msg>Remove post step which deleted the SImon sfdsi because issue is resolved see #266.<commit_after>#!/usr/bin/env python
# Thomas Reerink
#
# Run example:
#  python generate-ec-earth-namelists-for-all-MIP-experiments.py
#
# Looping over all MIPs and within each MIP over all its MIP experiments.
# The experiment tier can be selected. For each selected experiment the
# namelists are created by calling the generate-ec-earth-namelists.sh script.
#

import sys
import os
from dreqPy import dreq
dq = dreq.loadDreq()

# Specify which tier experiments should be included:
experiment_tiers_included = 1
#ec_earth_mips = ['CMIP'] # for basic test
ec_earth_mips = ['CMIP', 'DCPP']
#ec_earth_mips = ['CMIP', 'AerChemMIP', 'C4MIP',          'DAMIP', 'DCPP',                              'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP',         'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] # All 18 EC-Earth MIPs
#ec_earth_mips = ['CMIP', 'AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] # All 23 CMIP6 MIPs
experiment_counter = 0

command_0 = 'rm -rf cmip6-output-control-files'
os.system(command_0)

# Loop over MIPs:
for mip in dq.coll['mip'].items:
  # Loop over experiments:
  for u in dq.inx.iref_by_sect[mip.uid].a['experiment']:
    ex = dq.inx.uid[u]
    if ex.label == 'esm-hist' or ex.label == 'esm-piControl':
     print 'Skipping this esm experiment ' + ex.label + ' because its CMIP6 data request fails so far.\n'
    else:
     if experiment_counter == 0:
       command = './generate-ec-earth-namelists.sh ' + mip.label + ' ' + ex.label + ' ' + str(ex.tier[0]) + ' 1'
     else:
       command = './generate-ec-earth-namelists.sh ' + mip.label + ' ' + ex.label + ' ' + str(ex.tier[0]) + ' 1 omit-setup'
     command_2 = 'rm -rf cmip6-output-control-files/' + mip.label + '/cmip6-experiment-*/file_def-compact'
     command_3 = 'rm -f  cmip6-output-control-files/' + mip.label + '/cmip6-experiment-*/cmip6-file_def_nemo.xml'
     command_4 = "sed -i -e 's/True\" field_ref=\"toce_pot\"/False\" field_ref=\"toce_pot\"/' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-' + mip.label + '-' + ex.label + '/file_def_nemo-opa.xml'
     command_5 = "sed -i -e '/sfdsi_2/d' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-' + mip.label + '-' + ex.label + '/file_def_nemo-opa.xml'
    #command_4 = "sed -i -e 's/True\" field_ref=\"toce_pot\"/False\" field_ref=\"toce_pot\"/' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-m=' + mip.label + '-e=' + ex.label + '-t=' + str(ex.tier[0]) + '-p=1/file_def_nemo-opa.xml'
    #command_5 = "sed -i -e '/sfdsi_2/d' cmip6-output-control-files/" + mip.label + '/cmip6-experiment-m=' + mip.label + '-e=' + ex.label + '-t=' + str(ex.tier[0]) + '-p=1/file_def_nemo-opa.xml'
    #print print '{}'.format(command)
     if mip.label in ec_earth_mips:
       #if ex.tier[0] == experiment_tiers_included and mip.label in ec_earth_mips and ex.label == 'piControl':  # for basic test
        if ex.tier[0] == experiment_tiers_included and mip.label in ec_earth_mips:
           os.system(command)
           os.system(command_2)
           os.system(command_3)
          #os.system(command_4) # Just set the toce fields false again because we still face troubles with them
          #os.system(command_5) # Delete the line with sfdsi_2 from the file_def_nemo-opa.xml files
           experiment_counter = experiment_counter + 1
        else:
           print ' Tier {:2} experiments are not included: Skipping: {}'.format(ex.tier[0], command)
     else:
        print ' EC-Earth3 does not participate in {:2}: Skipping: {}'.format(mip.label, command)

print ' There are {} experiments included. '.format(experiment_counter)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import sys
import json
import argparse
import glob
from munch import Munch
import re
from ortools.linear_solver import pywraplp
import math
from pathlib import Path

parser = argparse.ArgumentParser()
parser.add_argument('-b', '--bins', required=True)
parser.add_argument('-d', '--durations', required=True)
parser.add_argument('-s', '--slow', default=False, action='store_true')
parser.add_argument('--beta')
args = parser.parse_args()

class NoTestError(Exception):
  pass
class FailedError(Exception):
  pass
class Tests:
  def __init__(self):
    self.tests = []

  def load(self, timings):
    with open(timings) as f:
      timings = json.load(f, object_hook=Munch.fromDict)

    tests = {}

    for feature in timings:
      if not 'elements' in feature: continue

      for test in feature.elements:
        if test.type == 'background': continue

        # for retries, the last successful iteration (if any) will overwrite the failed iterations
        name = re.sub(r' -- @[0-9]+\.[0-9]+ ', '', test.name)
        tests[name] = Munch(
          name = name,
          duration = sum([step.result.duration for step in test.steps if 'result' in step and 'duration' in step.result]),
          failed = (test.status == 'failed'),
          slow = 'use.with_slow=true' in test.tags or 'slow' in test.tags
        )
    if len(tests) == 0: raise NoTestError()
    if any(1 for test in tests.values() if test.failed): raise FailedError()

    self.tests += tests.values()

  def balance(self):
    tests = [test for test in self.tests if not test.slow or args.slow]
    self.duration = sum([test.duration for test in tests])
    solver = pywraplp.Solver.CreateSolver('SCIP')

    data = Munch(
      weights = [test.duration for test in tests],
      tests = list(range(len(tests))),
      bins = list(range(len(tests))),
      bin_capacity = math.ceil(max([test.duration for test in tests]))
    )
    # https://developers.google.com/optimization/bin/bin_packing
    # x[i, j] = 1 if item i is packed in bin j.
    x = {
      (i, j): solver.IntVar(0, 1, f'x_{i}_{j}')
      for i in data.tests
      for j in data.bins
    }
    # y[j] = 1 if bin j is used.
    y = {
      j: solver.IntVar(0, 1, 'y[%i]' % j)
      for j in data.bins
    }
    # constraints
    # Each item must be in exactly one bin.
    for i in data.tests:
      solver.Add(sum(x[i, j] for j in data.bins) == 1)
    # The amount packed in each bin cannot exceed its capacity.
    for j in data.bins:
      solver.Add(sum(x[(i, j)] * data.weights[i] for i in data.tests) <= y[j] * data.bin_capacity)
    # Objective: minimize the number of bins used.
    solver.Minimize(solver.Sum([y[j] for j in data.bins]))

    solution = solver.Solve()
    if solution != pywraplp.Solver.OPTIMAL:
      raise ValueError('No optimal solution')
    bins = {}
    for j in data.bins:
      if y[j].solution_value() == 1:
        bin_tests = []
        bin_weight = 0
        for i in data.tests:
          if x[i, j].solution_value() > 0:
            bin_tests.append(tests[i])
            bin_weight += data.weights[i]
        if bin_weight > 0:
          bins[j] = bin_tests
    bins = sorted(bins.values(), key=lambda cluster: sum([test.duration for test in cluster]))
    print('Time: ', math.ceil(solver.WallTime()/ 1000), 'seconds')
    print('Bins:', [sum([test.duration for test in cluster]) for cluster in bins])

    Path(os.path.dirname(args.bins)).mkdir(parents=True, exist_ok=True)
    with open(args.bins, 'w') as f:
      json.dump([[test.name for test in cluster] for cluster in bins], f, indent='  ')

Tests = Tests()
Tests.load(args.durations)
Tests.balance()
print(f"::set-output name=bins::{json.dumps(list(range(len(bins))))}")

print('beta:', args.beta)
clients = ['zotero', 'jurism']
if args.beta == 'true':
  clients += [client + '-beta' for client in clients]
  print('### REMOVING jurism-beta ###')
  clients = [client for client in clients if client != 'jurism-beta']
print(f"::set-output name=bins::{json.dumps(clients)}")
<commit_msg>active balance<commit_after>#!/usr/bin/env python3

import os
import sys
import json
import argparse
import glob
from munch import Munch
import re
from ortools.linear_solver import pywraplp
import math
from pathlib import Path

parser = argparse.ArgumentParser()
parser.add_argument('-b', '--bins', required=True)
parser.add_argument('-d', '--durations', required=True)
parser.add_argument('-s', '--slow', default=False, action='store_true')
parser.add_argument('--beta')
args = parser.parse_args()

class NoTestError(Exception):
  pass
class FailedError(Exception):
  pass
class Tests:
  def __init__(self):
    self.tests = []

  def load(self, timings):
    with open(timings) as f:
      timings = json.load(f, object_hook=Munch.fromDict)

    tests = {}

    for feature in timings:
      if not 'elements' in feature: continue

      for test in feature.elements:
        if test.type == 'background': continue

        # for retries, the last successful iteration (if any) will overwrite the failed iterations
        name = re.sub(r' -- @[0-9]+\.[0-9]+ ', '', test.name)
        tests[name] = Munch(
          name = name,
          duration = sum([step.result.duration for step in test.steps if 'result' in step and 'duration' in step.result]),
          failed = (test.status == 'failed'),
          slow = 'use.with_slow=true' in test.tags or 'slow' in test.tags
        )
    if len(tests) == 0: raise NoTestError()
    if any(1 for test in tests.values() if test.failed): raise FailedError()

    self.tests += tests.values()

  def balance(self):
    tests = [test for test in self.tests if not test.slow or args.slow]
    self.duration = sum([test.duration for test in tests])
    solver = pywraplp.Solver.CreateSolver('SCIP')

    data = Munch(
      weights = [test.duration for test in tests],
      tests = list(range(len(tests))),
      bins = list(range(len(tests))),
      bin_capacity = math.ceil(max([test.duration for test in tests]))
    )
    # https://developers.google.com/optimization/bin/bin_packing
    # x[i, j] = 1 if item i is packed in bin j.
    x = {
      (i, j): solver.IntVar(0, 1, f'x_{i}_{j}')
      for i in data.tests
      for j in data.bins
    }
    # y[j] = 1 if bin j is used.
    y = {
      j: solver.IntVar(0, 1, 'y[%i]' % j)
      for j in data.bins
    }
    # constraints
    # Each item must be in exactly one bin.
    for i in data.tests:
      solver.Add(sum(x[i, j] for j in data.bins) == 1)
    # The amount packed in each bin cannot exceed its capacity.
    for j in data.bins:
      solver.Add(sum(x[(i, j)] * data.weights[i] for i in data.tests) <= y[j] * data.bin_capacity)
    # Objective: minimize the number of bins used.
    solver.Minimize(solver.Sum([y[j] for j in data.bins]))

    solution = solver.Solve()
    if solution != pywraplp.Solver.OPTIMAL:
      raise ValueError('No optimal solution')
    bins = {}
    for j in data.bins:
      if y[j].solution_value() == 1:
        bin_tests = []
        bin_weight = 0
        for i in data.tests:
          if x[i, j].solution_value() > 0:
            bin_tests.append(tests[i])
            bin_weight += data.weights[i]
        if bin_weight > 0:
          bins[j] = bin_tests
    self.bins = sorted(bins.values(), key=lambda cluster: sum([test.duration for test in cluster]))
    print('Time: ', math.ceil(solver.WallTime()/ 1000), 'seconds')
    print('Bins:', [sum([test.duration for test in cluster]) for cluster in self.bins])

    Path(os.path.dirname(args.bins)).mkdir(parents=True, exist_ok=True)
    with open(args.bins, 'w') as f:
      json.dump([[test.name for test in cluster] for cluster in self.bins], f, indent='  ')

Tests = Tests()
Tests.load(args.durations)
Tests.balance()
print(f"::set-output name=bins::{json.dumps(list(range(len(Tests.bins))))}")

print('beta:', args.beta)
clients = ['zotero', 'jurism']
if args.beta == 'true':
  clients += [client + '-beta' for client in clients]
  print('### REMOVING jurism-beta ###')
  clients = [client for client in clients if client != 'jurism-beta']
print(f"::set-output name=bins::{json.dumps(clients)}")
<|endoftext|>


<commit_before>from PIL import Image
import sys, os, platform

QR_DIR = '.'
try:
    b = u'\u2588'.encode(sys.stdin.encoding)
    sys.stdout.write(b + '\r')
    sys.stdout.flush()
except:
    BLOCK = 'MM'
else:
    BLOCK = b

class QRCode():
    def __init__(self, fileName, size, padding = 0, background = 'BLACK'):
        self.size = size
        self.padding = padding
        self.img = Image.open(fileName)
        self.times = self.img.size[0]/(size + padding * 2)
        self.rgb = self.img.convert('RGB')
        self.white = BLOCK if background == 'BLACK' else '  '
        self.black = '  ' if background == 'BLACK' else BLOCK
    def print_qr(self):
        sys.stdout.write(' '*50 + '\r')
        sys.stdout.flush()
        print(self.white * (self.size + 2))
        startPoint = self.padding + 0.5
        for y in range(self.size):
            sys.stdout.write(self.white)
            for x in range(self.size):
                r,g,b = self.rgb.getpixel(((x + startPoint) * self.times, (y + startPoint) * self.times))
                sys.stdout.write(self.white if r > 127 else self.black)
            print(self.white)
        print(self.white * (self.size + 2))

if __name__ == '__main__':
    # 37 is for picture size without padding, 3 is padding
    q = QRCode(os.path.join(QR_DIR, 'QR.jpg'), 37, 3, 'BLACK')
    q.print_qr()
<commit_msg>Optimize py3 compatibility & py2 run speed<commit_after>from PIL import Image
import sys, os

QR_DIR = '.'
try:
    b = u'\u2588'
    sys.stdout.write(b + '\r')
    sys.stdout.flush()
except UnicodeEncodeError:
    BLOCK = 'MM'
else:
    BLOCK = b

class QRCode():
    def __init__(self, fileName, size, padding = 0, background = 'BLACK'):
        self.size = size
        self.padding = padding
        self.img = Image.open(fileName)
        self.times = self.img.size[0]/(size + padding * 2)
        self.rgb = self.img.convert('RGB')
        self.white = BLOCK if background == 'BLACK' else '  '
        self.black = '  ' if background == 'BLACK' else BLOCK
    def print_qr(self):
        sys.stdout.write(' '*50 + '\r')
        sys.stdout.flush()
        qr = self.white * (self.size + 2) + '\n'
        startPoint = self.padding + 0.5
        for y in range(self.size):
            qr += self.white
            for x in range(self.size):
                r,g,b = self.rgb.getpixel(((x + startPoint) * self.times, (y + startPoint) * self.times))
                qr += self.white if r > 127 else self.black
            qr += self.white + '\n'
        qr += self.white * (self.size + 2) + '\n'
        sys.stdout.write(qr)

if __name__ == '__main__':
    # 37 is for picture size without padding, 3 is padding
    q = QRCode(os.path.join(QR_DIR, 'QR.jpg'), 37, 3, 'BLACK')
    q.print_qr()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-

# Documentation available here:
# http://www.trackvis.org/docs/?subsect=fileformat

import os
import logging
import numpy as np

from tractconverter.formats import header
from tractconverter.formats.header import Header as H

def readBinaryBytes(f, nbBytes, dtype):
    buff = f.read(nbBytes * dtype.itemsize)
    return np.frombuffer(buff, dtype=dtype)

class TRK:
    MAGIC_NUMBER = "TRACK"
    OFFSET = 1000
    # self.hdr
    # self.filename
    # self.hdr[H.ENDIAN]
    # self.FIBER_DELIMITER
    # self.END_DELIMITER

    #####
    # Static Methods
    ###
    @staticmethod
    def _check(filename):
        f = open(filename, 'rb')
        magicNumber = f.read(5)
        f.close()
        return magicNumber == TRK.MAGIC_NUMBER

    @staticmethod
    def create(filename, hdr, anatFile=None):
        f = open(filename, 'wb')
        f.write(TRK.MAGIC_NUMBER + "\n")
        f.close()

        trk = TRK(filename, load=False)
        trk.hdr = hdr
        trk.writeHeader()

        return trk

    #####
    # Methods
    ###
    def __init__(self, filename, anatFile=None, load=True):
        if not TRK._check(filename):
            raise NameError("Not a TRK file.")

        self.filename = filename
        self.hdr = {}
        if load:
            self._load()
            self.hdr = header.get_header_from_anat(anatFile, self.hdr)

    def _load(self):
        f = open(self.filename, 'rb')

        #####
        # Read header
        ###
        self.hdr[H.MAGIC_NUMBER] = f.read(6)
        self.hdr[H.DIMENSIONS] = np.frombuffer(f.read(6), dtype='<i2')
        self.hdr[H.VOXEL_SIZES] = np.frombuffer(f.read(12), dtype='<f4')
        self.hdr[H.ORIGIN] = np.frombuffer(f.read(12), dtype='<f4')
        self.hdr[H.NB_SCALARS_BY_POINT] = np.frombuffer(f.read(2), dtype='<i2')[0]
        self.hdr['scalar_name'] = [f.read(20) for i in range(10)]
        self.hdr[H.NB_PROPERTIES_BY_TRACT] = np.frombuffer(f.read(2), dtype='<i2')[0]
        self.hdr['property_name'] = [f.read(20) for i in range(10)]

        self.hdr[H.VOXEL_TO_WORLD] = np.frombuffer(f.read(64), dtype='<f4').reshape(4, 4)
        self.hdr[H.WORLD_ORDER] = "RAS"

        # Skip reserved bytes
        f.seek(444, os.SEEK_CUR)

        self.hdr[H.VOXEL_ORDER] = f.read(4)
        self.hdr["pad2"] = f.read(4)
        self.hdr["image_orientation_patient"] = np.frombuffer(f.read(24), dtype='<f4')
        self.hdr["pad1"] = f.read(2)

        self.hdr["invert_x"] = f.read(1) == '\x01'
        self.hdr["invert_y"] = f.read(1) == '\x01'
        self.hdr["invert_z"] = f.read(1) == '\x01'
        self.hdr["swap_xy"] = f.read(1) == '\x01'
        self.hdr["swap_yz"] = f.read(1) == '\x01'
        self.hdr["swap_zx"] = f.read(1) == '\x01'

        self.hdr[H.NB_FIBERS] = np.frombuffer(f.read(4), dtype='<i4')
        self.hdr["version"] = np.frombuffer(f.read(4), dtype='<i4')
        self.hdr["hdr_size"] = np.frombuffer(f.read(4), dtype='<i4')

        # Check if little or big endian
        self.hdr[H.ENDIAN] = '<'
        if self.hdr["hdr_size"] != self.OFFSET:
            self.hdr[H.ENDIAN] = '>'
            self.hdr[H.NB_FIBERS] = self.hdr[H.NB_FIBERS].astype('>i4')
            self.hdr["version"] = self.hdr["version"].astype('>i4')
            self.hdr["hdr_size"] = self.hdr["hdr_size"].astype('>i4')

        nb_fibers = 0
        self.hdr[H.NB_POINTS] = 0

        #Either verify the number of streamlines specified in the header is correct or
        # count the actual number of streamlines in case it is not specified in the header.
        remainingBytes = os.path.getsize(self.filename) - self.OFFSET
        while remainingBytes > 0:
            # Read points
            nbPoints = readBinaryBytes(f, 1, np.dtype(self.hdr[H.ENDIAN] + "i4"))[0]
            self.hdr[H.NB_POINTS] += nbPoints
            # This seek is used to go to the next points number indication in the file.
            f.seek((nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT])
                   + self.hdr[H.NB_PROPERTIES_BY_TRACT]) * 4, 1)  # Relative seek
            remainingBytes -= (nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT])
                               + self.hdr[H.NB_PROPERTIES_BY_TRACT]) * 4 + 4
            nb_fibers += 1

        if self.hdr[H.NB_FIBERS] != nb_fibers:
            logging.warn('The number of streamlines specified in header ({1}) does not match ' +
                         'the actual number of streamlines contained in this file ({1}). ' +
                         'The latter will be used.'.format(self.hdr[H.NB_FIBERS], nb_fibers))

        self.hdr[H.NB_FIBERS] = nb_fibers

        f.close()

    def writeHeader(self):
        # Get the voxel size and format it as an array.
        voxel_sizes = np.asarray(self.hdr.get(H.VOXEL_SIZES, (1.0, 1.0, 1.0)), dtype='<f4')
        dimensions = np.asarray(self.hdr.get(H.DIMENSIONS, (0, 0, 0)), dtype='<i2')
        voxel2world = np.asarray(self.hdr.get(H.VOXEL_TO_WORLD, np.eye(4)), dtype='<f4')

        f = open(self.filename, 'wb')
        f.write(self.MAGIC_NUMBER + "\0")   # id_string
        f.write(dimensions)                 # dim
        f.write(voxel_sizes)                # voxel_size
        f.write(np.zeros(12, dtype='i1'))   # origin
        f.write(np.zeros(2, dtype='i1'))    # n_scalars
        f.write(np.zeros(200, dtype='i1'))  # scalar_name
        f.write(np.zeros(2, dtype='i1'))    # n_properties
        f.write(np.zeros(200, dtype='i1'))  # property_name
        f.write(voxel2world)                # vos_to_ras
        f.write(np.zeros(444, dtype='i1'))  # reserved
        f.write(np.zeros(4, dtype='i1'))    # voxel_order
        f.write(np.zeros(4, dtype='i1'))    # pad2
        f.write(np.zeros(24, dtype='i1'))   # image_orientation_patient
        f.write(np.zeros(2, dtype='i1'))    # pad1
        f.write(np.zeros(1, dtype='i1'))    # invert_x
        f.write(np.zeros(1, dtype='i1'))    # invert_y
        f.write(np.zeros(1, dtype='i1'))    # invert_z
        f.write(np.zeros(1, dtype='i1'))    # swap_xy
        f.write(np.zeros(1, dtype='i1'))    # swap_yz
        f.write(np.zeros(1, dtype='i1'))    # swap_zx
        f.write(np.array(self.hdr[H.NB_FIBERS], dtype='<i4'))
        f.write(np.array([2], dtype='<i4'))  # version
        f.write(np.array(self.OFFSET, dtype='<i4'))  # hdr_size, should be 1000
        f.close()

    def close(self):
        pass

    def __iadd__(self, fibers):
        f = open(self.filename, 'ab')
        for fib in fibers:
            f.write(np.array([len(fib)], '<i4').tostring())
            f.write(fib.astype("<f4").tostring())
        f.close()

        return self

    #####
    # Iterate through fibers
    ###
    def __iter__(self):
        if self.hdr[H.NB_FIBERS] == 0:
            return

        f = open(self.filename, 'rb')
        f.seek(self.OFFSET)

        remainingBytes = os.path.getsize(self.filename) - self.OFFSET

        cpt = 0
        while cpt < self.hdr[H.NB_FIBERS] or remainingBytes > 0:
            # Read points
            nbPoints = readBinaryBytes(f, 1, np.dtype(self.hdr[H.ENDIAN] + "i4"))[0]
            ptsAndScalars = readBinaryBytes(f,
                                            nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT]),
                                            np.dtype(self.hdr[H.ENDIAN] + "f4"))

            # If there are some properties, ignore them for now.
            properties = readBinaryBytes(f,
                                         self.hdr[H.NB_PROPERTIES_BY_TRACT],
                                         np.dtype(self.hdr[H.ENDIAN] + "f4"))

            newShape = [-1, 3 + self.hdr[H.NB_SCALARS_BY_POINT]]
            ptsAndScalars = ptsAndScalars.reshape(newShape)

            pointsWithoutScalars = ptsAndScalars[:, 0:3]
            yield pointsWithoutScalars

            remainingBytes -= 4  # Number of points
            remainingBytes -= nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT]) * 4
            # For now, we do not process the tract properties, so just skip over them.
            remainingBytes -= self.hdr[H.NB_PROPERTIES_BY_TRACT] * 4
            cpt += 1

        f.close()

    def __str__(self):
        text = ""
        text += "MAGIC NUMBER: {0}".format(self.hdr[H.MAGIC_NUMBER])
        text += "\nv.{0}".format(self.hdr['version'])
        text += "\ndim: {0}".format(self.hdr[H.DIMENSIONS])
        text += "\nvoxel_sizes: {0}".format(self.hdr[H.VOXEL_SIZES])
        text += "\norigin: {0}".format(self.hdr[H.ORIGIN])
        text += "\nnb_scalars: {0}".format(self.hdr[H.NB_SCALARS_BY_POINT])
        text += "\nscalar_name:\n{0}".format("\n".join(self.hdr['scalar_name']))
        text += "\nnb_properties: {0}".format(self.hdr[H.NB_PROPERTIES_BY_TRACT])
        text += "\nproperty_name:\n{0}".format("\n".join(self.hdr['property_name']))
        text += "\nvox_to_world:\n{0}".format(self.hdr[H.VOXEL_TO_WORLD])
        text += "\nworld_order: {0}".format(self.hdr[H.WORLD_ORDER])
        text += "\nvoxel_order: {0}".format(self.hdr[H.VOXEL_ORDER])
        text += "\nimage_orientation_patient: {0}".format(self.hdr['image_orientation_patient'])
        text += "\npad1: {0}".format(self.hdr['pad1'])
        text += "\npad2: {0}".format(self.hdr['pad2'])
        text += "\ninvert_x: {0}".format(self.hdr['invert_x'])
        text += "\ninvert_y: {0}".format(self.hdr['invert_y'])
        text += "\ninvert_z: {0}".format(self.hdr['invert_z'])
        text += "\nswap_xy: {0}".format(self.hdr['swap_xy'])
        text += "\nswap_yz: {0}".format(self.hdr['swap_yz'])
        text += "\nswap_zx: {0}".format(self.hdr['swap_zx'])
        text += "\nn_count: {0}".format(self.hdr[H.NB_FIBERS])
        text += "\nhdr_size: {0}".format(self.hdr['hdr_size'])
        text += "\nendianess: {0}".format(self.hdr[H.ENDIAN])

        return text
<commit_msg>Overwrite voxel order when anatomy is provided.<commit_after># -*- coding: UTF-8 -*-

# Documentation available here:
# http://www.trackvis.org/docs/?subsect=fileformat

import os
import logging
import numpy as np
import nibabel as nib

from tractconverter.formats import header
from tractconverter.formats.header import Header as H

def readBinaryBytes(f, nbBytes, dtype):
    buff = f.read(nbBytes * dtype.itemsize)
    return np.frombuffer(buff, dtype=dtype)

class TRK:
    MAGIC_NUMBER = "TRACK"
    OFFSET = 1000
    # self.hdr
    # self.filename
    # self.hdr[H.ENDIAN]
    # self.FIBER_DELIMITER
    # self.END_DELIMITER

    #####
    # Static Methods
    ###
    @staticmethod
    def _check(filename):
        f = open(filename, 'rb')
        magicNumber = f.read(5)
        f.close()
        return magicNumber == TRK.MAGIC_NUMBER

    @staticmethod
    def create(filename, hdr, anatFile=None):
        f = open(filename, 'wb')
        f.write(TRK.MAGIC_NUMBER + "\n")
        f.close()

        trk = TRK(filename, load=False)
        trk.hdr = hdr
        trk.writeHeader()

        return trk

    #####
    # Methods
    ###
    def __init__(self, filename, anatFile=None, load=True):
        if not TRK._check(filename):
            raise NameError("Not a TRK file.")

        self.filename = filename
        self.hdr = {}
        if load:
            self._load()
            self.hdr = header.get_header_from_anat(anatFile, self.hdr)

    def _load(self):
        f = open(self.filename, 'rb')

        #####
        # Read header
        ###
        self.hdr[H.MAGIC_NUMBER] = f.read(6)
        self.hdr[H.DIMENSIONS] = np.frombuffer(f.read(6), dtype='<i2')
        self.hdr[H.VOXEL_SIZES] = np.frombuffer(f.read(12), dtype='<f4')
        self.hdr[H.ORIGIN] = np.frombuffer(f.read(12), dtype='<f4')
        self.hdr[H.NB_SCALARS_BY_POINT] = np.frombuffer(f.read(2), dtype='<i2')[0]
        self.hdr['scalar_name'] = [f.read(20) for i in range(10)]
        self.hdr[H.NB_PROPERTIES_BY_TRACT] = np.frombuffer(f.read(2), dtype='<i2')[0]
        self.hdr['property_name'] = [f.read(20) for i in range(10)]

        self.hdr[H.VOXEL_TO_WORLD] = np.frombuffer(f.read(64), dtype='<f4').reshape(4, 4)
        self.hdr[H.WORLD_ORDER] = "RAS"

        # Skip reserved bytes
        f.seek(444, os.SEEK_CUR)

        self.hdr[H.VOXEL_ORDER] = f.read(4)
        self.hdr["pad2"] = f.read(4)
        self.hdr["image_orientation_patient"] = np.frombuffer(f.read(24), dtype='<f4')
        self.hdr["pad1"] = f.read(2)

        self.hdr["invert_x"] = f.read(1) == '\x01'
        self.hdr["invert_y"] = f.read(1) == '\x01'
        self.hdr["invert_z"] = f.read(1) == '\x01'
        self.hdr["swap_xy"] = f.read(1) == '\x01'
        self.hdr["swap_yz"] = f.read(1) == '\x01'
        self.hdr["swap_zx"] = f.read(1) == '\x01'

        self.hdr[H.NB_FIBERS] = np.frombuffer(f.read(4), dtype='<i4')
        self.hdr["version"] = np.frombuffer(f.read(4), dtype='<i4')
        self.hdr["hdr_size"] = np.frombuffer(f.read(4), dtype='<i4')

        # Check if little or big endian
        self.hdr[H.ENDIAN] = '<'
        if self.hdr["hdr_size"] != self.OFFSET:
            self.hdr[H.ENDIAN] = '>'
            self.hdr[H.NB_FIBERS] = self.hdr[H.NB_FIBERS].astype('>i4')
            self.hdr["version"] = self.hdr["version"].astype('>i4')
            self.hdr["hdr_size"] = self.hdr["hdr_size"].astype('>i4')

        nb_fibers = 0
        self.hdr[H.NB_POINTS] = 0

        #Either verify the number of streamlines specified in the header is correct or
        # count the actual number of streamlines in case it is not specified in the header.
        remainingBytes = os.path.getsize(self.filename) - self.OFFSET
        while remainingBytes > 0:
            # Read points
            nbPoints = readBinaryBytes(f, 1, np.dtype(self.hdr[H.ENDIAN] + "i4"))[0]
            self.hdr[H.NB_POINTS] += nbPoints
            # This seek is used to go to the next points number indication in the file.
            f.seek((nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT])
                   + self.hdr[H.NB_PROPERTIES_BY_TRACT]) * 4, 1)  # Relative seek
            remainingBytes -= (nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT])
                               + self.hdr[H.NB_PROPERTIES_BY_TRACT]) * 4 + 4
            nb_fibers += 1

        if self.hdr[H.NB_FIBERS] != nb_fibers:
            logging.warn('The number of streamlines specified in header ({1}) does not match ' +
                         'the actual number of streamlines contained in this file ({1}). ' +
                         'The latter will be used.'.format(self.hdr[H.NB_FIBERS], nb_fibers))

        self.hdr[H.NB_FIBERS] = nb_fibers

        f.close()

    def writeHeader(self):
        # Get the voxel size and format it as an array.
        voxel_sizes = np.asarray(self.hdr.get(H.VOXEL_SIZES, (1.0, 1.0, 1.0)), dtype='<f4')
        dimensions = np.asarray(self.hdr.get(H.DIMENSIONS, (0, 0, 0)), dtype='<i2')
        voxel2world = np.asarray(self.hdr.get(H.VOXEL_TO_WORLD, np.eye(4)), dtype='<f4')
        voxel_order = np.asarray(self.hdr.get(H.VOXEL_ORDER, ''.join(nib.aff2axcodes(voxel2world))), dtype='S4')

        f = open(self.filename, 'wb')
        f.write(self.MAGIC_NUMBER + "\0")   # id_string
        f.write(dimensions)                 # dim
        f.write(voxel_sizes)                # voxel_size
        f.write(np.zeros(12, dtype='i1'))   # origin
        f.write(np.zeros(2, dtype='i1'))    # n_scalars
        f.write(np.zeros(200, dtype='i1'))  # scalar_name
        f.write(np.zeros(2, dtype='i1'))    # n_properties
        f.write(np.zeros(200, dtype='i1'))  # property_name
        f.write(voxel2world)                # vos_to_ras
        f.write(np.zeros(444, dtype='i1'))  # reserved
        f.write(voxel_order)                # voxel_order
        f.write(np.zeros(4, dtype='i1'))    # pad2
        f.write(np.zeros(24, dtype='i1'))   # image_orientation_patient
        f.write(np.zeros(2, dtype='i1'))    # pad1
        f.write(np.zeros(1, dtype='i1'))    # invert_x
        f.write(np.zeros(1, dtype='i1'))    # invert_y
        f.write(np.zeros(1, dtype='i1'))    # invert_z
        f.write(np.zeros(1, dtype='i1'))    # swap_xy
        f.write(np.zeros(1, dtype='i1'))    # swap_yz
        f.write(np.zeros(1, dtype='i1'))    # swap_zx
        f.write(np.array(self.hdr[H.NB_FIBERS], dtype='<i4'))
        f.write(np.array([2], dtype='<i4'))  # version
        f.write(np.array(self.OFFSET, dtype='<i4'))  # hdr_size, should be 1000
        f.close()

    def close(self):
        pass

    def __iadd__(self, fibers):
        f = open(self.filename, 'ab')
        for fib in fibers:
            f.write(np.array([len(fib)], '<i4').tostring())
            f.write(fib.astype("<f4").tostring())
        f.close()

        return self

    #####
    # Iterate through fibers
    ###
    def __iter__(self):
        if self.hdr[H.NB_FIBERS] == 0:
            return

        f = open(self.filename, 'rb')
        f.seek(self.OFFSET)

        remainingBytes = os.path.getsize(self.filename) - self.OFFSET

        cpt = 0
        while cpt < self.hdr[H.NB_FIBERS] or remainingBytes > 0:
            # Read points
            nbPoints = readBinaryBytes(f, 1, np.dtype(self.hdr[H.ENDIAN] + "i4"))[0]
            ptsAndScalars = readBinaryBytes(f,
                                            nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT]),
                                            np.dtype(self.hdr[H.ENDIAN] + "f4"))

            # If there are some properties, ignore them for now.
            properties = readBinaryBytes(f,
                                         self.hdr[H.NB_PROPERTIES_BY_TRACT],
                                         np.dtype(self.hdr[H.ENDIAN] + "f4"))

            newShape = [-1, 3 + self.hdr[H.NB_SCALARS_BY_POINT]]
            ptsAndScalars = ptsAndScalars.reshape(newShape)

            pointsWithoutScalars = ptsAndScalars[:, 0:3]
            yield pointsWithoutScalars

            remainingBytes -= 4  # Number of points
            remainingBytes -= nbPoints * (3 + self.hdr[H.NB_SCALARS_BY_POINT]) * 4
            # For now, we do not process the tract properties, so just skip over them.
            remainingBytes -= self.hdr[H.NB_PROPERTIES_BY_TRACT] * 4
            cpt += 1

        f.close()

    def __str__(self):
        text = ""
        text += "MAGIC NUMBER: {0}".format(self.hdr[H.MAGIC_NUMBER])
        text += "\nv.{0}".format(self.hdr['version'])
        text += "\ndim: {0}".format(self.hdr[H.DIMENSIONS])
        text += "\nvoxel_sizes: {0}".format(self.hdr[H.VOXEL_SIZES])
        text += "\norigin: {0}".format(self.hdr[H.ORIGIN])
        text += "\nnb_scalars: {0}".format(self.hdr[H.NB_SCALARS_BY_POINT])
        text += "\nscalar_name:\n{0}".format("\n".join(self.hdr['scalar_name']))
        text += "\nnb_properties: {0}".format(self.hdr[H.NB_PROPERTIES_BY_TRACT])
        text += "\nproperty_name:\n{0}".format("\n".join(self.hdr['property_name']))
        text += "\nvox_to_world:\n{0}".format(self.hdr[H.VOXEL_TO_WORLD])
        text += "\nworld_order: {0}".format(self.hdr[H.WORLD_ORDER])
        text += "\nvoxel_order: {0}".format(self.hdr[H.VOXEL_ORDER])
        text += "\nimage_orientation_patient: {0}".format(self.hdr['image_orientation_patient'])
        text += "\npad1: {0}".format(self.hdr['pad1'])
        text += "\npad2: {0}".format(self.hdr['pad2'])
        text += "\ninvert_x: {0}".format(self.hdr['invert_x'])
        text += "\ninvert_y: {0}".format(self.hdr['invert_y'])
        text += "\ninvert_z: {0}".format(self.hdr['invert_z'])
        text += "\nswap_xy: {0}".format(self.hdr['swap_xy'])
        text += "\nswap_yz: {0}".format(self.hdr['swap_yz'])
        text += "\nswap_zx: {0}".format(self.hdr['swap_zx'])
        text += "\nn_count: {0}".format(self.hdr[H.NB_FIBERS])
        text += "\nhdr_size: {0}".format(self.hdr['hdr_size'])
        text += "\nendianess: {0}".format(self.hdr[H.ENDIAN])

        return text
<|endoftext|>


<commit_before># coding: utf-8
# This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013 Zachary Spector,  zacharyspector@gmail.com
"""Data to initialize the database with"""

def _(arg):
    """Nothing!"""
    return arg

whole_imgrows = [
    ('default_wallpaper', 'LiSE/gui/assets/wallpape.jpg', 0),
    ('default_spot', 'LiSE/gui/assets/orb.png', 0),
    ('default_pawn', 'atlas://LiSE/gui/assets/rltiles/hominid/base/unseen', 0),
    ('locked', 'LiSE/gui/assets/locked.png', 0),
    ('unlocked', 'LiSE/gui/assets/unlocked.png', 0)
]
"""Special imgs that need their own names, irrespective of the
user's wishes."""

graphics = {
    'locked': {'imgs': ['locked']},
    'unlocked': {'imgs': ['unlocked']},
    'default_wallpaper': {
        'imgs': ['default_wallpaper']
    },
    'default_spot': {
        'imgs': ['default_spot']
    },
    'default_pawn': {
        'offset_x': 4,
        'offset_y': 8,
        'imgs': ['default_pawn']
    }
}
"""Special graphics"""

globs = [
    ('branch',   'int', '0'),
    ('tick',     'int', '0'),
    ('language', 'unicode', 'eng'),
    ('observer', 'unicode', 'Omniscient'),
    ('observed', 'unicode', 'Player'),
    ('host',     'unicode', 'Physical'),
    ('top_generic_graphic', 'int', '0'),
    ('default_facade_cls', 'unicode', 'NullFacade')
]
"""Default values for globals"""

stackhs = [
    (10, ('block', 'brutalist')),
    (
        6,  (
            'crossroad',
            'corporate',
            'modernist',
            'brownstone',
            'gray',
            'lobby',
            'bunker',
            'red',
            'orange'
        )
    ),
           (5,  ('4sidewalk', 'street-ne-sw', 'street-nw-se')),
           (4, ('spacer',))]
"""Stacking heights, presently only used for putting Pixel City tiles
one on the other"""

offys = [
    (-2, ('spacer',)),
    (1,  ('enterprise', 'block')),
    (-1, ('lobby', 'street-ne-sw', 'street-nw-se'))
]
"""Y-offsets for certain imgs"""

offxs = [(1, ('lobby', 'modernist', 'orange'))]
"""X-offsets for certain imgs"""

reciprocal_portals = []
"""Portals that connect two places both ways"""

one_way_portals = []
"""Portals that connect one place to another place and only in that
direction"""

spot_coords = []
"""Pairs of spots with their coordinates"""

things = {}
"""Thing names mapped to their locations"""

pawns = {}
"""Thing names mapped to their graphics"""
<commit_msg>The functions that insert initial data into the database live here now<commit_after># coding: utf-8
# This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013 Zachary Spector,  zacharyspector@gmail.com
"""Data to initialize the database with"""

def _(arg):
    """Nothing!"""
    return arg

whole_imgrows = [
    ('default_wallpaper', 'LiSE/gui/assets/wallpape.jpg', 0),
    ('default_spot', 'LiSE/gui/assets/orb.png', 0),
    ('default_pawn', 'atlas://LiSE/gui/assets/rltiles/hominid/base/unseen', 0),
    ('locked', 'LiSE/gui/assets/locked.png', 0),
    ('unlocked', 'LiSE/gui/assets/unlocked.png', 0)
]
"""Special imgs that need their own names, irrespective of the
user's wishes."""

graphics = {
    'locked': {'imgs': ['locked']},
    'unlocked': {'imgs': ['unlocked']},
    'default_wallpaper': {
        'imgs': ['default_wallpaper']
    },
    'default_spot': {
        'imgs': ['default_spot']
    },
    'default_pawn': {
        'offset_x': 4,
        'offset_y': 8,
        'imgs': ['default_pawn']
    }
}
"""Special graphics"""

globs = [
    ('branch',   'int', '0'),
    ('tick',     'int', '0'),
    ('language', 'unicode', 'eng'),
    ('observer', 'unicode', 'Omniscient'),
    ('observed', 'unicode', 'Player'),
    ('host',     'unicode', 'Physical'),
    ('top_generic_graphic', 'int', '0'),
    ('default_facade_cls', 'unicode', 'NullFacade')
]
"""Default values for globals"""

stackhs = [
    (10, ('block', 'brutalist')),
    (
        6,  (
            'crossroad',
            'corporate',
            'modernist',
            'brownstone',
            'gray',
            'lobby',
            'bunker',
            'red',
            'orange'
        )
    ),
           (5,  ('4sidewalk', 'street-ne-sw', 'street-nw-se')),
           (4, ('spacer',))]
"""Stacking heights, presently only used for putting Pixel City tiles
one on the other"""

offys = [
    (-2, ('spacer',)),
    (1,  ('enterprise', 'block')),
    (-1, ('lobby', 'street-ne-sw', 'street-nw-se'))
]
"""Y-offsets for certain imgs"""

offxs = [(1, ('lobby', 'modernist', 'orange'))]
"""X-offsets for certain imgs"""

reciprocal_portals = []
"""Portals that connect two places both ways"""

one_way_portals = []
"""Portals that connect one place to another place and only in that
direction"""

spot_coords = []
"""Pairs of spots with their coordinates"""

things = {}
"""Thing names mapped to their locations"""

pawns = {}
"""Thing names mapped to their graphics"""

def defaults(c, kivy=False):
    """Retrieve default values from ``LiSE.data`` and insert them with the
    cursor ``c``.

    With ``kivy``==``True``, this will include data about graphics."""
    if kivy:
        c.executemany(
            "INSERT INTO img (name, path, stacking_height) "
            "VALUES (?, ?, ?);",
            whole_imgrows
        )
        for (name, d) in graphics.iteritems():
            c.execute(
                "INSERT INTO graphic (name, offset_x, offset_y) "
                "VALUES (?, ?, ?);",
                (name, d.get('offset_x', 0), d.get('offset_y', 0))
            )
            for i in xrange(0, len(d['imgs'])):
                c.execute(
                    "INSERT INTO graphic_img (graphic, layer, img) "
                    "VALUES (?, ?, ?);",
                    (name, i, d['imgs'][i])
                )
        for (height, names) in stackhs:
            qrystr = (
                "UPDATE img SET stacking_height=? WHERE name IN ({});".format(
                    ", ".join(["?"] * len(names))
                )
            )
            qrytup = (height,) + names
            c.execute(qrystr, qrytup)
    c.executemany(
        "INSERT INTO globals (key, type, value) VALUES (?, ?, ?);",
        globs
    )
    c.execute(
        "INSERT INTO timestream (branch, parent) VALUES (?, ?);",
        (0, 0)
    )
    for character in things:
        for thing in things[character]:
            c.execute(
                "INSERT INTO thing (character, name, host) VALUES (?, ?, ?);",
                (character, thing, things[character][thing]["host"])
            )
            c.execute(
                "INSERT INTO thing_loc (character, name, location) "
                "VALUES (?, ?, ?);",
                (character, thing, things[character][thing]["location"])
            )
    from LiSE.data import reciprocal_portals
    for (orig, dest) in reciprocal_portals:
        name1 = "{}->{}".format(orig, dest)
        name2 = "{}->{}".format(dest, orig)
        c.executemany(
            "INSERT INTO portal (name) VALUES (?);",
            [(name1,), (name2,)]
        )
        c.executemany(
            "INSERT INTO portal_loc (name, origin, destination) VALUES "
            "(?, ?, ?);", [(name1, orig, dest), (name2, dest, orig)]
        )
    from LiSE.data import one_way_portals
    for (orig, dest) in one_way_portals:
        name = "{}->{}".format(orig, dest)
        c.execute(
            "INSERT INTO portal (name) VALUES (?);",
            (name,)
        )
        c.execute(
            "INSERT INTO portal_loc (name, origin, destination) "
            "VALUES (?, ?, ?);", (name, orig, dest)
        )

def mkdb(DB_NAME, lisepath, kivy=False):
    """Initialize a database file and insert default values.

    """
    import sqlite3
    img_qrystr = (
        "INSERT INTO img (name, path) "
        "VALUES (?, ?);"
    )
    tag_qrystr = (
        "INSERT INTO img_tag (img, tag) VALUES (?, ?);"
    )

    def ins_atlas(curs, path, qualify=False, tags=[]):
        """Grab all the images in an atlas and store them, optionally sticking
        the name of the atlas on the start.

        Apply the given tags if any.

        """
        from kivy.atlas import Atlas
        from os import sep
        lass = Atlas(path)
        atlaspath = "atlas://{}".format(path[:-6])
        atlasn = path.split(sep)[-1][:-6]
        for tilen in lass.textures.iterkeys():
            imgn = atlasn + '.' + tilen if qualify else tilen
            curs.execute(
                img_qrystr, (
                    imgn, "{}/{}".format(atlaspath, tilen)
                )
            )
            for tag in tags:
                curs.execute(tag_qrystr, (imgn, tag))

    def ins_atlas_dir(curs, dirname, qualify=False, tags=[]):
        from os import sep
        from os import listdir
        """Recurse into the directory and ins_atlas for all atlas therein."""
        for fn in listdir(dirname):
            if fn[-5:] == 'atlas':
                path = dirname + sep + fn
                ins_atlas(curs, path, qualify, [fn[:-6]] + tags)

    if kivy:
        import kivy.logger
        Logger = kivy.logger.Logger
        # I just need these modules to fill in the relevant bits of
        # SaveableMetaclass, which they do when imported. They don't
        # have to do anything else, so delete them.
        import LiSE.gui.img
        del LiSE.gui.img
        import LiSE.gui.board
        del LiSE.gui.board
    else:
        import logging
        logging.basicConfig()
        Logger = logging.getLogger()
        Logger.setLevel(0)

    conn = sqlite3.connect(DB_NAME)
    c = conn.cursor()

    Logger.debug("initializing database")
    from LiSE.orm import SaveableMetaclass
    # contains classes with table declarations in them.

    Logger.debug("inserting default values")
    defaults(c, kivy)

    if kivy:
        Logger.debug("indexing the RLTiles")
        ins_atlas_dir(
            c,
            "LiSE/gui/assets/rltiles/hominid",
            True,
            ['hominid', 'rltile', 'pawn']
        )

        Logger.debug("indexing Pixel City")
        ins_atlas(
            c,
            "LiSE/gui/assets/pixel_city.atlas",
            False,
            ['spot', 'pixel_city']
        )

    conn.commit()
    return conn
<|endoftext|>