<commit_before>#!/usr/bin/env python

from addict import Dict
from circus.client import CircusClient
from circus import get_arbiter

class Client(object):

    def __init__(self, host, port, timeout=15):
        assert(type(host), str)
        assert(type(port), int)
        assert(type(timeout), int)
        self._host = host
        self._port = port
        self._timeout = timeout
        self._arbiter = get_arbiter([])
        self._arbiter.start()
        self._client = CircusClient(timeout=self._timeout, endpoint='{0}:{1}'.format(self._host, self._port))

    """
    Add a watcher:
        This command add a watcher dynamically to a arbiter
    """
    def addWatcher(self, name, command, args=[], autostart=False):
        assert(type(name), str)
        assert(type(command), str)
        assert(type(args), list)
        assert(type(autostart), bool)

        addWatcher_command = Dict()
        addWatcher_command.command = 'add'
        addWatcher_command.properties.cmd = command
        addWatcher_command.properties.name = name
        addWatcher_command.properties.args = args
        addWatcher_command.properties.start = autostart

        response = self._client.call(addWatcher_command)
        response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Reload the arbiter or a watcher:
        This command reloads all the process in a watcher or all watchers
    """
    def reload(self, watcher='', graceful=True, sequential=False, waiting=False):
        assert(type(watcher), str)
        assert(type(graceful), bool)
        assert(type(sequential), bool)
        assert(type(waiting), bool)

    """
    Restart the arbiter or a watcher:
        This command restart all the process in a watcher or all watchers
    """
    def restart(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

    """
    Remove a watcher:
        This command removes a watcher dynamically from the arbiter
    """
    def rmWatcher(self, watcher, nonstop=False, waiting=False):
        assert(type(watcher), str)
        assert(type(nonstop), bool)
        assert(type(waiting), bool)

    """
    Get the number of watchers:
        Get the number of watchers in a arbiter
    """
    def numWatchers(self):
        pass

    """
        Get list of watchers or processes in a watcher
    """
    def list(self, watcher=''):
        assert(type(watcher), str)

    """
    Start the arbiter or a watcher:
        This command starts all the processes in a watcher or all watchers.
    """
    def start(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

    """
    Stop watchers:
        This command stops a given watcher or all watchers.
    """
    def stop(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

    """
    Get the number of processes:
        Get the number of processes in a watcher or in a arbiter
    """
    def numProcesses(self, watcher):
        assert(type(watcher), str)

    """
    Quit the arbiter immediately:
        When the arbiter receive this command, the arbiter exit.
    """
    def quit(self, waiting=False):
        assert(type(waiting), bool)
<commit_msg>Basic commands<commit_after>#!/usr/bin/env python

from addict import Dict
from circus.client import CircusClient
from circus import get_arbiter

class Client(object):

    def __init__(self, host, port, timeout=15):
        assert(type(host), str)
        assert(type(port), int)
        assert(type(timeout), int)
        self._host = host
        self._port = port
        self._timeout = timeout
        self._arbiter = get_arbiter([])
        self._arbiter.start()
        self._client = CircusClient(timeout=self._timeout, endpoint='{0}:{1}'.format(self._host, self._port))

    """
    Add a watcher:
        This command add a watcher dynamically to a arbiter
    """
    def addWatcher(self, name, command, args=[], autostart=False):
        assert(type(name), str)
        assert(type(command), str)
        assert(type(args), list)
        assert(type(autostart), bool)

        addWatcher_command = Dict()
        addWatcher_command.command = 'add'
        addWatcher_command.properties.cmd = command
        addWatcher_command.properties.name = name
        addWatcher_command.properties.args = args
        addWatcher_command.properties.start = autostart

        response = self._client.call(addWatcher_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Reload the arbiter or a watcher:
        This command reloads all the process in a watcher or all watchers
    """
    def reload(self, watcher='', graceful=True, sequential=False, waiting=False):
        assert(type(watcher), str)
        assert(type(graceful), bool)
        assert(type(sequential), bool)
        assert(type(waiting), bool)

        reload_command = Dict()
        reload_command.command = 'reload'
        reload_command.properties.name = watcher
        reload_command.properties.graceful = graceful
        reload_command.properties.sequential = sequential
        reload_command.properties.waiting = waiting

        response = self._client.call(reload_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Restart the arbiter or a watcher:
        This command restart all the process in a watcher or all watchers
    """
    def restart(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

        restart_command = Dict()
        restart_command.command = 'restart'
        restart_command.properties.name = watcher
        restart_command.properties.waiting = waiting

        response = self._client.call(restart_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Remove a watcher:
        This command removes a watcher dynamically from the arbiter
    """
    def rmWatcher(self, watcher, nonstop=False, waiting=False):
        assert(type(watcher), str)
        assert(type(nonstop), bool)
        assert(type(waiting), bool)

        rm_command = Dict()
        rm_command.command = 'rm'
        rm_command.properties.name = watcher
        rm_command.properties.nonstop = nonstop
        rm_command.properties.waiting = waiting

        response = self._client.call(rm_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Get the number of watchers:
        Get the number of watchers in a arbiter
    """
    def numWatchers(self):
        numwatchers_command = Dict()
        numwatchers_command.command = 'numwatchers'

        response = self._client.call(numwatchers_command)
        status = response.get('status', None)

        if status:
            if status == 'ok':
                return response.get('numwatchers', 0)
        return 0

    """
        Get list of watchers or processes in a watcher
    """
    def list(self, watcher=''):
        assert(type(watcher), str)

        list_command = Dict()
        list_command.command = 'list'

        if watcher:
            list_command.properties.name = watcher

        response = self._client.call(list_command)
        return [{'name':str(w)} for w in response['watchers']] if response['status'] == u'ok' else []

    """
    Start the arbiter or a watcher:
        This command starts all the processes in a watcher or all watchers.
    """
    def start(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

        start_command = Dict()
        start_command.command = 'start'
        start_command.properties.name = watcher
        start_command.properties.waiting = waiting

        response = self._client.call(start_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Stop watchers:
        This command stops a given watcher or all watchers.
    """
    def stop(self, watcher='', waiting=False):
        assert(type(watcher), str)
        assert(type(waiting), bool)

        stop_command = Dict()
        stop_command.command = 'stop'
        stop_command.properties.name = watcher
        stop_command.properties.waiting = waiting

        response = self._client.call(stop_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False

    """
    Get the number of processes:
        Get the number of processes in a watcher or in a arbiter
    """
    def numProcesses(self, watcher):
        assert(type(watcher), str)

        num_command = Dict()
        num_command.command = 'numprocesses'
        num_command.properties.name = watcher

        response = self._client.call(num_command)
        status = response.get('status', None)

        if status:
            if status == 'ok':
                return response.get('numprocesses', 0)
        return 0

    """
    Quit the arbiter immediately:
        When the arbiter receive this command, the arbiter exit.
    """
    def quit(self, waiting=False):
        assert(type(waiting), bool)

        quit_command = Dict()
        quit_command.command = 'quit'
        quit_command.waiting = waiting

        response = self._client.call(quit_command)
        response = response.get('status', None)

        if response:
            if response == 'ok':
                return True
        return False
<|endoftext|>


<commit_before>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""
import collections
import os
import traceback
from albow import FloatField, IntField, AttrRef, Row, Label, Widget, TabPanel, CheckBox, Column, Button, TextFieldWrapped, TextField
from editortools.blockview import BlockButton
from editortools.editortool import EditorTool
from glbackground import Panel
from mceutils import ChoiceButton, alertException, setWindowCaption, showProgress, TextInputRow
import mcplatform
from operation import Operation
from albow.dialogs import wrapped_label, alert
import pymclevel
from pymclevel import BoundingBox

def alertFilterException(func):
    def _func(*args, **kw):
        try:
            func(*args, **kw)
        except Exception, e:
            print traceback.format_exc()
            alert(u"Exception during filter operation. See console for details.\n\n{0}".format(e))

    return _func

def addNumField(page, optionName, val, min=None, max=None):
        if isinstance(val, float):
            ftype = FloatField
        else:
            ftype = IntField

        if min == max:
            min = None
            max = None

        field = ftype(value=val, width=100, min=min, max=max)
        page.optionDict[optionName] = AttrRef(field, 'value')

        row = Row([Label(optionName), field])
        return row

class FilterModuleOptions(Widget):
    is_gl_container = True

    def __init__(self, tool, module, *args, **kw):
        Widget.__init__(self, *args, **kw)
        self.tool = tool
        pages = TabPanel()
        pages.is_gl_container = True
        self.pages = pages
        self.optionDict = {}
        pageTabContents = []

        self.giveEditorObject(module)
        print "Creating options for ", module
        if hasattr(module, "inputs"):
            if isinstance(module.inputs, list):
                for tabData in module.inputs:
                    title, page, pageRect = self.makeTabPage(self.tool, tabData)
                    pages.add_page(title, page)
                    pages.set_rect(pageRect.union(pages._rect))
            elif isinstance(module.inputs, tuple):
                title, page, pageRect = self.makeTabPage(self.tool, module.inputs)
                pages.add_page(title, page)
                pages.set_rect(pageRect)
        else:
            self.size = (0, 0)

        pages.shrink_wrap()
        self.add(pages)
        self.shrink_wrap()
        if len(pages.pages):
            if(pages.current_page != None):
                pages.show_page(pages.current_page)
            else:
                pages.show_page(pages.pages[0])

        for eachPage in pages.pages:
            self.optionDict = dict(self.optionDict.items() + eachPage.optionDict.items())

    def makeTabPage(self, tool, inputs):
        page = Widget()
        page.is_gl_container = True
        rows = []
        cols = []
        height = 0
        max_height = 550
        page.optionDict = {}
        page.tool = tool
        title = "Tab"

        for optionName, optionType in inputs:
            if isinstance(optionType, tuple):
                if isinstance(optionType[0], (int, long, float)):
                    if len(optionType) > 2:
                        val, min, max = optionType
                    elif len(optionType) == 2:
                        min, max = optionType
                        val = min

                    rows.append(addNumField(page, optionName, val, min, max))

                if isinstance(optionType[0], (str, unicode)):
                    isChoiceButton = False

                    if optionType[0] == "string":           #This code seems to be completely dead.
                        kwds = []
                        wid = None
                        lin = None
                        val = None
                        for keyword in optionType:
                            if isinstance(keyword, (str, unicode)) and keyword != "string":
                                kwds.append(keyword)
                        for keyword in kwds:
                            splitWord = keyword.split('=')
                            if len(splitWord) > 1:
                                v = None
                                key = None

                                try:
                                    v = int(splitWord[1])
                                except:
                                    pass

                                key = splitWord[0]
                                if v is not None:
                                    if key == "lines":
                                        lin = v
                                    elif key == "width":
                                        wid = v
                                else:
                                    if key == "value":
                                        val = splitWord[1]

                        if lin is None:
                            lin = 1
                        if val is None:
                            val = "Input String Here"
                        if wid is None:
                            wid = 200

                        field = TextField(value=val, width=wid, lines=lin)
                        page.optionDict[optionName] = AttrRef(field, 'value')

                        row = Row((Label(optionName), field))
                        rows.append(row)
                    else:
                        isChoiceButton = True

                    if isChoiceButton:
                        choiceButton = ChoiceButton(map(str, optionType))
                        page.optionDict[optionName] = AttrRef(choiceButton, 'selectedChoice')

                        rows.append(Row((Label(optionName), choiceButton)))

            elif isinstance(optionType, bool):
                cbox = CheckBox(value=optionType)
                page.optionDict[optionName] = AttrRef(cbox, 'value')

                row = Row((Label(optionName), cbox))
                rows.append(row)

            elif isinstance(optionType, (int, float)):
                rows.append(addNumField(self, optionName, optionType))

            elif optionType == "blocktype" or isinstance(optionType, pymclevel.materials.Block):
                blockButton = BlockButton(tool.editor.level.materials)
                if isinstance(optionType, pymclevel.materials.Block):
                    blockButton.blockInfo = optionType

                row = Column((Label(optionName), blockButton))
                page.optionDict[optionName] = AttrRef(blockButton, 'blockInfo')

                rows.append(row)
            elif optionType == "label":
                rows.append(wrapped_label(optionName, 50))

            elif optionType == "string":
                input = None #not sure how to pull values from filters, but leaves it open for the future. Use this variable to set field width.
                if input != None:
                    size = input
                else:
                    size = 200
                field = TextField(value="Input String Here")
                row = TextInputRow(optionName, ref=AttrRef(field, 'value'), width = size)
                page.optionDict[optionName] = AttrRef(field, 'value')
                rows.append(row)

            elif optionType == "title":
                title = optionName

            else:
                raise ValueError(("Unknown option type", optionType))

        height = sum(r.height for r in rows)

        if height > max_height:
            h = 0
            for i, r in enumerate(rows):
                h += r.height
                if h > height / 2:
                    break

            cols.append(Column(rows[:i]))
            rows = rows[i:]
        #cols.append(Column(rows))

        if len(rows):
            cols.append(Column(rows))

        if len(cols):
            page.add(Row(cols))
        page.shrink_wrap()

        return (title, page, page._rect)

    @property
    def options(self):
        return dict((k, v.get()) for k, v in self.optionDict.iteritems())

    @options.setter
    def options(self, val):
        for k in val:
            if k in self.optionDict:
                self.optionDict[k].set(val[k])

    def giveEditorObject(self, module):
        module.editor = self.tool.editor

class FilterToolPanel(Panel):
    def __init__(self, tool):
        Panel.__init__(self)

        self.savedOptions = {}

        self.tool = tool
        self.selectedFilterName = None
        if len(self.tool.filterModules):
            self.reload()

    def reload(self):
        for i in list(self.subwidgets):
            self.remove(i)

        tool = self.tool

        if len(tool.filterModules) is 0:
            self.add(Label("No filter modules found!"))
            self.shrink_wrap()
            return

        if self.selectedFilterName is None or self.selectedFilterName not in tool.filterNames:
            self.selectedFilterName = tool.filterNames[0]

        self.filterOptionsPanel = None
        while self.filterOptionsPanel is None:
            module = self.tool.filterModules[self.selectedFilterName]
            try:
                self.filterOptionsPanel = FilterModuleOptions(self.tool, module)
            except Exception, e:
                alert("Error creating filter inputs for {0}: {1}".format(module, e))
                traceback.print_exc()
                self.tool.filterModules.pop(self.selectedFilterName)
                self.selectedFilterName = tool.filterNames[0]

            if len(tool.filterNames) == 0:
                raise ValueError("No filters loaded!")

        self.filterSelect = ChoiceButton(tool.filterNames, choose=self.filterChanged)
        self.filterSelect.selectedChoice = self.selectedFilterName

        self.confirmButton = Button("Filter", action=self.tool.confirm)

        filterLabel = Label("Filter:", fg_color=(177, 177, 255, 255))
        filterLabel.mouse_down = lambda x: mcplatform.platform_open(mcplatform.filtersDir)
        filterLabel.tooltipText = "Click to open filters folder"
        filterSelectRow = Row((filterLabel, self.filterSelect))

        self.add(Column((filterSelectRow, self.filterOptionsPanel, self.confirmButton)))

        self.shrink_wrap()
        if self.parent:
            self.centery = self.parent.centery

        if self.selectedFilterName in self.savedOptions:
            self.filterOptionsPanel.options = self.savedOptions[self.selectedFilterName]

    def filterChanged(self):
        self.saveOptions()
        self.selectedFilterName = self.filterSelect.selectedChoice
        self.reload()

    filterOptionsPanel = None

    def saveOptions(self):
        if self.filterOptionsPanel:
            self.savedOptions[self.selectedFilterName] = self.filterOptionsPanel.options

class FilterOperation(Operation):
    def __init__(self, editor, level, box, filter, options):
        super(FilterOperation, self).__init__(editor, level)
        self.box = box
        self.filter = filter
        self.options = options

    def perform(self, recordUndo=True):
        if recordUndo:
            self.undoLevel = self.extractUndo(self.level, self.box)

        self.filter.perform(self.level, BoundingBox(self.box), self.options)

        pass

    def dirtyBox(self):
        return self.box

class FilterTool(EditorTool):
    tooltipText = "Filter"
    toolIconName = "filter"

    def __init__(self, editor):
        EditorTool.__init__(self, editor)

        self.filterModules = {}

        self.panel = FilterToolPanel(self)

    @property
    def statusText(self):
        return "Choose a filter, then click Filter or press ENTER to apply it."

    def toolEnabled(self):
        return not (self.selectionBox() is None)

    def toolSelected(self):
        self.showPanel()

    @alertException
    def showPanel(self):
        if self.panel.parent:
            self.editor.remove(self.panel)

        self.reloadFilters()

        #self.panel = FilterToolPanel(self)
        self.panel.reload()

        self.panel.left = self.editor.left
        self.panel.centery = self.editor.centery

        self.editor.add(self.panel)

    def hidePanel(self):
        self.panel.saveOptions()
        if self.panel.parent:
            self.panel.parent.remove(self.panel)

    def reloadFilters(self):
        filterDir = mcplatform.filtersDir
        filterFiles = os.listdir(filterDir)
        filterPyfiles = filter(lambda x: x.endswith(".py"), filterFiles)

        def tryImport(name):
            try:
                return __import__(name)
            except Exception, e:
                print traceback.format_exc()
                alert(u"Exception while importing filter module {}. See console for details.\n\n{}".format(name, e))
                return object()

        filterModules = (tryImport(x[:-3]) for x in filterPyfiles)
        filterModules = filter(lambda module: hasattr(module, "perform"), filterModules)

        self.filterModules = collections.OrderedDict(sorted((self.moduleDisplayName(x), x) for x in filterModules))
        for m in self.filterModules.itervalues():
            try:
                reload(m)
            except Exception, e:
                print traceback.format_exc()
                alert(u"Exception while reloading filter module {}. Using previously loaded module. See console for details.\n\n{}".format(m.__file__, e))

    @property
    def filterNames(self):
        return [self.moduleDisplayName(module) for module in self.filterModules.itervalues()]

    def moduleDisplayName(self, module):
        return module.displayName if hasattr(module, 'displayName') else module.__name__.capitalize()

    @alertFilterException
    def confirm(self):

        with setWindowCaption("APPLYING FILTER - "):
            filterModule = self.filterModules[self.panel.filterSelect.selectedChoice]

            op = FilterOperation(self.editor, self.editor.level, self.selectionBox(), filterModule, self.panel.filterOptionsPanel.options)

            self.editor.level.showProgress = showProgress

            self.editor.addOperation(op)
            self.editor.addUnsavedEdit()

            self.editor.invalidateBox(self.selectionBox())
<commit_msg>Change filter strings to be blank by default<commit_after>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""
import collections
import os
import traceback
from albow import FloatField, IntField, AttrRef, Row, Label, Widget, TabPanel, CheckBox, Column, Button, TextFieldWrapped, TextField
from editortools.blockview import BlockButton
from editortools.editortool import EditorTool
from glbackground import Panel
from mceutils import ChoiceButton, alertException, setWindowCaption, showProgress, TextInputRow
import mcplatform
from operation import Operation
from albow.dialogs import wrapped_label, alert
import pymclevel
from pymclevel import BoundingBox

def alertFilterException(func):
    def _func(*args, **kw):
        try:
            func(*args, **kw)
        except Exception, e:
            print traceback.format_exc()
            alert(u"Exception during filter operation. See console for details.\n\n{0}".format(e))

    return _func

def addNumField(page, optionName, val, min=None, max=None):
        if isinstance(val, float):
            ftype = FloatField
        else:
            ftype = IntField

        if min == max:
            min = None
            max = None

        field = ftype(value=val, width=100, min=min, max=max)
        page.optionDict[optionName] = AttrRef(field, 'value')

        row = Row([Label(optionName), field])
        return row

class FilterModuleOptions(Widget):
    is_gl_container = True

    def __init__(self, tool, module, *args, **kw):
        Widget.__init__(self, *args, **kw)
        self.tool = tool
        pages = TabPanel()
        pages.is_gl_container = True
        self.pages = pages
        self.optionDict = {}
        pageTabContents = []

        self.giveEditorObject(module)
        print "Creating options for ", module
        if hasattr(module, "inputs"):
            if isinstance(module.inputs, list):
                for tabData in module.inputs:
                    title, page, pageRect = self.makeTabPage(self.tool, tabData)
                    pages.add_page(title, page)
                    pages.set_rect(pageRect.union(pages._rect))
            elif isinstance(module.inputs, tuple):
                title, page, pageRect = self.makeTabPage(self.tool, module.inputs)
                pages.add_page(title, page)
                pages.set_rect(pageRect)
        else:
            self.size = (0, 0)

        pages.shrink_wrap()
        self.add(pages)
        self.shrink_wrap()
        if len(pages.pages):
            if(pages.current_page != None):
                pages.show_page(pages.current_page)
            else:
                pages.show_page(pages.pages[0])

        for eachPage in pages.pages:
            self.optionDict = dict(self.optionDict.items() + eachPage.optionDict.items())

    def makeTabPage(self, tool, inputs):
        page = Widget()
        page.is_gl_container = True
        rows = []
        cols = []
        height = 0
        max_height = 550
        page.optionDict = {}
        page.tool = tool
        title = "Tab"

        for optionName, optionType in inputs:
            if isinstance(optionType, tuple):
                if isinstance(optionType[0], (int, long, float)):
                    if len(optionType) > 2:
                        val, min, max = optionType
                    elif len(optionType) == 2:
                        min, max = optionType
                        val = min

                    rows.append(addNumField(page, optionName, val, min, max))

                if isinstance(optionType[0], (str, unicode)):
                    isChoiceButton = False

                    if optionType[0] == "string":           #This code seems to be completely dead.
                        kwds = []
                        wid = None
                        lin = None
                        val = None
                        for keyword in optionType:
                            if isinstance(keyword, (str, unicode)) and keyword != "string":
                                kwds.append(keyword)
                        for keyword in kwds:
                            splitWord = keyword.split('=')
                            if len(splitWord) > 1:
                                v = None
                                key = None

                                try:
                                    v = int(splitWord[1])
                                except:
                                    pass

                                key = splitWord[0]
                                if v is not None:
                                    if key == "lines":
                                        lin = v
                                    elif key == "width":
                                        wid = v
                                else:
                                    if key == "value":
                                        val = splitWord[1]

                        if lin is None:
                            lin = 1
                        if val is None:
                            val = "Input String Here"
                        if wid is None:
                            wid = 200

                        field = TextField(value=val, width=wid, lines=lin)
                        page.optionDict[optionName] = AttrRef(field, 'value')

                        row = Row((Label(optionName), field))
                        rows.append(row)
                    else:
                        isChoiceButton = True

                    if isChoiceButton:
                        choiceButton = ChoiceButton(map(str, optionType))
                        page.optionDict[optionName] = AttrRef(choiceButton, 'selectedChoice')

                        rows.append(Row((Label(optionName), choiceButton)))

            elif isinstance(optionType, bool):
                cbox = CheckBox(value=optionType)
                page.optionDict[optionName] = AttrRef(cbox, 'value')

                row = Row((Label(optionName), cbox))
                rows.append(row)

            elif isinstance(optionType, (int, float)):
                rows.append(addNumField(self, optionName, optionType))

            elif optionType == "blocktype" or isinstance(optionType, pymclevel.materials.Block):
                blockButton = BlockButton(tool.editor.level.materials)
                if isinstance(optionType, pymclevel.materials.Block):
                    blockButton.blockInfo = optionType

                row = Column((Label(optionName), blockButton))
                page.optionDict[optionName] = AttrRef(blockButton, 'blockInfo')

                rows.append(row)
            elif optionType == "label":
                rows.append(wrapped_label(optionName, 50))

            elif optionType == "string":
                input = None #not sure how to pull values from filters, but leaves it open for the future. Use this variable to set field width.
                if input != None:
                    size = input
                else:
                    size = 200
                field = TextField(value="")
                row = TextInputRow(optionName, ref=AttrRef(field, 'value'), width = size)
                page.optionDict[optionName] = AttrRef(field, 'value')
                rows.append(row)

            elif optionType == "title":
                title = optionName

            else:
                raise ValueError(("Unknown option type", optionType))

        height = sum(r.height for r in rows)

        if height > max_height:
            h = 0
            for i, r in enumerate(rows):
                h += r.height
                if h > height / 2:
                    break

            cols.append(Column(rows[:i]))
            rows = rows[i:]
        #cols.append(Column(rows))

        if len(rows):
            cols.append(Column(rows))

        if len(cols):
            page.add(Row(cols))
        page.shrink_wrap()

        return (title, page, page._rect)

    @property
    def options(self):
        return dict((k, v.get()) for k, v in self.optionDict.iteritems())

    @options.setter
    def options(self, val):
        for k in val:
            if k in self.optionDict:
                self.optionDict[k].set(val[k])

    def giveEditorObject(self, module):
        module.editor = self.tool.editor

class FilterToolPanel(Panel):
    def __init__(self, tool):
        Panel.__init__(self)

        self.savedOptions = {}

        self.tool = tool
        self.selectedFilterName = None
        if len(self.tool.filterModules):
            self.reload()

    def reload(self):
        for i in list(self.subwidgets):
            self.remove(i)

        tool = self.tool

        if len(tool.filterModules) is 0:
            self.add(Label("No filter modules found!"))
            self.shrink_wrap()
            return

        if self.selectedFilterName is None or self.selectedFilterName not in tool.filterNames:
            self.selectedFilterName = tool.filterNames[0]

        self.filterOptionsPanel = None
        while self.filterOptionsPanel is None:
            module = self.tool.filterModules[self.selectedFilterName]
            try:
                self.filterOptionsPanel = FilterModuleOptions(self.tool, module)
            except Exception, e:
                alert("Error creating filter inputs for {0}: {1}".format(module, e))
                traceback.print_exc()
                self.tool.filterModules.pop(self.selectedFilterName)
                self.selectedFilterName = tool.filterNames[0]

            if len(tool.filterNames) == 0:
                raise ValueError("No filters loaded!")

        self.filterSelect = ChoiceButton(tool.filterNames, choose=self.filterChanged)
        self.filterSelect.selectedChoice = self.selectedFilterName

        self.confirmButton = Button("Filter", action=self.tool.confirm)

        filterLabel = Label("Filter:", fg_color=(177, 177, 255, 255))
        filterLabel.mouse_down = lambda x: mcplatform.platform_open(mcplatform.filtersDir)
        filterLabel.tooltipText = "Click to open filters folder"
        filterSelectRow = Row((filterLabel, self.filterSelect))

        self.add(Column((filterSelectRow, self.filterOptionsPanel, self.confirmButton)))

        self.shrink_wrap()
        if self.parent:
            self.centery = self.parent.centery

        if self.selectedFilterName in self.savedOptions:
            self.filterOptionsPanel.options = self.savedOptions[self.selectedFilterName]

    def filterChanged(self):
        self.saveOptions()
        self.selectedFilterName = self.filterSelect.selectedChoice
        self.reload()

    filterOptionsPanel = None

    def saveOptions(self):
        if self.filterOptionsPanel:
            self.savedOptions[self.selectedFilterName] = self.filterOptionsPanel.options

class FilterOperation(Operation):
    def __init__(self, editor, level, box, filter, options):
        super(FilterOperation, self).__init__(editor, level)
        self.box = box
        self.filter = filter
        self.options = options

    def perform(self, recordUndo=True):
        if recordUndo:
            self.undoLevel = self.extractUndo(self.level, self.box)

        self.filter.perform(self.level, BoundingBox(self.box), self.options)

        pass

    def dirtyBox(self):
        return self.box

class FilterTool(EditorTool):
    tooltipText = "Filter"
    toolIconName = "filter"

    def __init__(self, editor):
        EditorTool.__init__(self, editor)

        self.filterModules = {}

        self.panel = FilterToolPanel(self)

    @property
    def statusText(self):
        return "Choose a filter, then click Filter or press ENTER to apply it."

    def toolEnabled(self):
        return not (self.selectionBox() is None)

    def toolSelected(self):
        self.showPanel()

    @alertException
    def showPanel(self):
        if self.panel.parent:
            self.editor.remove(self.panel)

        self.reloadFilters()

        #self.panel = FilterToolPanel(self)
        self.panel.reload()

        self.panel.left = self.editor.left
        self.panel.centery = self.editor.centery

        self.editor.add(self.panel)

    def hidePanel(self):
        self.panel.saveOptions()
        if self.panel.parent:
            self.panel.parent.remove(self.panel)

    def reloadFilters(self):
        filterDir = mcplatform.filtersDir
        filterFiles = os.listdir(filterDir)
        filterPyfiles = filter(lambda x: x.endswith(".py"), filterFiles)

        def tryImport(name):
            try:
                return __import__(name)
            except Exception, e:
                print traceback.format_exc()
                alert(u"Exception while importing filter module {}. See console for details.\n\n{}".format(name, e))
                return object()

        filterModules = (tryImport(x[:-3]) for x in filterPyfiles)
        filterModules = filter(lambda module: hasattr(module, "perform"), filterModules)

        self.filterModules = collections.OrderedDict(sorted((self.moduleDisplayName(x), x) for x in filterModules))
        for m in self.filterModules.itervalues():
            try:
                reload(m)
            except Exception, e:
                print traceback.format_exc()
                alert(u"Exception while reloading filter module {}. Using previously loaded module. See console for details.\n\n{}".format(m.__file__, e))

    @property
    def filterNames(self):
        return [self.moduleDisplayName(module) for module in self.filterModules.itervalues()]

    def moduleDisplayName(self, module):
        return module.displayName if hasattr(module, 'displayName') else module.__name__.capitalize()

    @alertFilterException
    def confirm(self):

        with setWindowCaption("APPLYING FILTER - "):
            filterModule = self.filterModules[self.panel.filterSelect.selectedChoice]

            op = FilterOperation(self.editor, self.editor.level, self.selectionBox(), filterModule, self.panel.filterOptionsPanel.options)

            self.editor.level.showProgress = showProgress

            self.editor.addOperation(op)
            self.editor.addUnsavedEdit()

            self.editor.invalidateBox(self.selectionBox())
<|endoftext|>


<commit_before>## @file intel_snb.py
# Post-processing for core and uncore SNB events are performed in this file
# for the pickler.
#
# The registers must be reported in the following order in the stats file:
# -# CTL registers
# -# Programmable CTR registers
# -# Fixed CTR registers
# -# If CTL register programming failed it (QPI) it seems to be 0, so this is added to event maps
# ---->>>> Programmable CTL registers must be in same order as CTR
import numpy

## Processor events
def CORE_PERF_EVENT(event_select, unit_mask):
    return event_select | (unit_mask << 8) | (1L << 16) | (1L << 17) | (1L << 21) | (1L << 22)
## Processor event map
cpu_event_map = {
    CORE_PERF_EVENT(0xD0,0x81) : 'LOAD_OPS_ALL,E',
    CORE_PERF_EVENT(0xD1,0x01) : 'LOAD_OPS_L1_HIT,E',
    CORE_PERF_EVENT(0xD1,0x02) : 'LOAD_OPS_L2_HIT,E',
    CORE_PERF_EVENT(0xD1,0x04) : 'LOAD_OPS_LLC_HIT,E',
    CORE_PERF_EVENT(0x10,0x90) : 'SSE_D_ALL,E',
    CORE_PERF_EVENT(0x11,0x02) : 'SIMD_D_256,E',
    CORE_PERF_EVENT(0xA2,0x01) : 'STALLS,E',
    CORE_PERF_EVENT(0x51,0x01) : 'LOAD_L1D_ALL,E',
    'FIXED0'                   : 'INSTRUCTIONS_RETIRED,E',
    'FIXED1'                   : 'CLOCKS_UNHALTED_CORE,E',
    'FIXED2'                   : 'CLOCKS_UNHALTED_REF,E',
}
## CBo events
def CBOX_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 17) | (0L << 18) | (0L << 19) | (1L << 22) | (0L << 23) | (1L << 24)
## CBo event map
cbo_event_map = {
    CBOX_PERF_EVENT(0x00, 0x00) : 'CLOCK_TICKS,E',
    CBOX_PERF_EVENT(0x11, 0x01) : 'RxR_OCCUPANCY,E',
    CBOX_PERF_EVENT(0x1F, 0x00) : 'COUNTER0_OCCUPANCY,E',
    CBOX_PERF_EVENT(0x34, 0x03) : 'LLC_LOOKUP,E',
    }
## Home Agent Unit events
def HAU_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L << 23) | (1L << 24)
## Home Agent map
hau_event_map = {
    HAU_PERF_EVENT(0x01, 0x03) : 'READ_REQUESTS,E',
    HAU_PERF_EVENT(0x01, 0x0C) : 'WRITE_REQUESTS,E',
    HAU_PERF_EVENT(0x00, 0x00) : 'CLOCKTICKS,E',
    HAU_PERF_EVENT(0x1A, 0x0F) : 'IMC_WRITES,E',
    }
## Integrated Memory events
def IMC_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24)
## Integrated Memory Controller map
imc_event_map = {
    IMC_PERF_EVENT(0x04, 0x03) : 'CAS_READS,E',
    IMC_PERF_EVENT(0x04, 0x0C) : 'CAS_WRITES,E',
    IMC_PERF_EVENT(0x01, 0x00) : 'ACT_COUNT,E',
    IMC_PERF_EVENT(0x02, 0x03) : 'PRE_COUNT_ALL,E',
    IMC_PERF_EVENT(0x02, 0x01) : 'PRE_COUNT_MISS,E',
    'FIXED0'                   : 'CYCLES,E',
    }
## Power Control Unit events
def PCU_PERF_EVENT(event):
    return (event) | (0L << 14) | (0L << 17) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24) | (0L << 31)
## Power Control map
pcu_event_map = {
    PCU_PERF_EVENT(0x06) : 'MAX_OS_CYCLES,E',
    PCU_PERF_EVENT(0x07) : 'MAX_CURRENT_CYCLES,E',
    PCU_PERF_EVENT(0x04) : 'MAX_TEMP_CYCLES,E',
    PCU_PERF_EVENT(0x05) : 'MAX_POWER_CYCLES,E',
    PCU_PERF_EVENT(0x81) : 'MIN_IO_CYCLES,E',
    PCU_PERF_EVENT(0x82) : 'MIN_SNOOP_CYCLES,E',
    'FIXED0'             : 'C3_CYCLES,E',
    'FIXED1'             : 'C6_CYCLES,E',
    }
## QPI Unit events
def QPI_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 15) | (0L << 18) | (1L << 21) | (1L << 22) | (0L <<23) | (1L << 24)
## QPI map
qpi_event_map = {
    QPI_PERF_EVENT(0x00, 0x01) : 'G0_IDLE,E',
    QPI_PERF_EVENT(0x00, 0x04) : 'G0_NON_DATA,E',
    QPI_PERF_EVENT(0x02, 0x08) : 'G1_DRS_DATA,E',
    QPI_PERF_EVENT(0x03, 0x04) : 'G2_NCB_DATA,E',
    }
## R2PCI Unit events
def R2PCI_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24)
## R2PCI map
r2pci_event_map = {
    R2PCI_PERF_EVENT(0x24, 0x04) : 'TRANSMITS,E',
    R2PCI_PERF_EVENT(0x01, 0x00) : 'CLOCKTICKS,E',
    R2PCI_PERF_EVENT(0x07, 0x0F) : 'ADDRESS_USED,E',
    R2PCI_PERF_EVENT(0x08, 0x0F) : 'ACKNOWLEDGED_USED,E',
    R2PCI_PERF_EVENT(0x09, 0x0F) : 'DATA_USED,E',
    }
## WESTMERE events
def WTM_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 16) | (1L << 17) | (1L <<21) | (1L << 22)
## WTM map
wtm_event_map = {
    WTM_PERF_EVENT(0x0F, 0x10) : 'MEM_UNCORE_RETIRED_REMOTE_DRAM,E',
    WTM_PERF_EVENT(0x0F, 0x20) : 'MEM_UNCORE_RETIRED_LOCAL_DRAM,E',
    WTM_PERF_EVENT(0x10, 0x01) : 'FP_COMP_OPS_EXE_X87,E',
    WTM_PERF_EVENT(0xCB, 0x01) : 'MEM_LOAD_RETIRED_L1D_HIT,E',
    'FIXED0'                   : 'INSTRUCTIONS_RETIRED,E',
    'FIXED1'                   : 'CLOCKS_UNHALTED_CORE,E',
    'FIXED2'                   : 'CLOCKS_UNHALTED_REF,E',
    }
## WESTMERE UNCORE events
def WTMUNC_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 22)
## WTM map
wtmunc_event_map = {
    WTMUNC_PERF_EVENT(0x08, 0x01) : 'L3_HITS_READ,E',
    WTMUNC_PERF_EVENT(0x08, 0x02) : 'L3_HITS_WRITE,E',
    WTMUNC_PERF_EVENT(0x08, 0x04) : 'L3_HITS_PROBE,E',
    WTMUNC_PERF_EVENT(0x09, 0x01) : 'L3_MISS_READ,E',
    WTMUNC_PERF_EVENT(0x09, 0x02) : 'L3_MISS_WRITE,E',
    WTMUNC_PERF_EVENT(0x09, 0x04) : 'L3_MISS_PROBE,E',
    WTMUNC_PERF_EVENT(0x0A, 0x0F) : 'L3_LINES_IN_ANY,E',
    WTMUNC_PERF_EVENT(0x0B, 0x1F) : 'L3_LINES_OUT_ANY,E',
    'FIXED0'                      : 'CLOCKS_UNCORE,E',
    }

## Reformats the counter arrays with configurable events
class reformat_counters:

    ## Constructor
    def __init__(self, job, name, event_map):
        ## Job Object
        self.job = job
        ## Name of device
        self.name = name
        ## List of CTL registers
        self.ctl_registers = []
        ## List of CTR registers
        self.ctr_registers = []
        ## List of Fixed registers
        self.fix_registers = []

        # Just need the first hosts schema
        for host in job.hosts.itervalues():
            if name not in host.stats: return
            stats = host.stats[name]
            break

        schema_desc = job.schemas.get(self.name).desc
        registers = schema_desc.split()

        for reg in registers:
            if reg.split(',')[1] != 'C':
                if reg.find('FIXED') == -1:
                    self.ctr_registers.append(registers.index(reg))
                else:
                    self.fix_registers.append(registers.index(reg))
            else:
                if reg.find('GLOBAL') == -1 and reg.find('FIXED') == -1 and reg.find('OPCODE_MATCH') == -1:
                    self.ctl_registers.append(registers.index(reg))

        # Build Schema from ctl registers and event maps
        dev_schema = []
        for dev, array in stats.iteritems():
            for j in self.ctl_registers:
                dev_schema.append(event_map.get(array[0,j],str(array[0,j])))
            break

        # Schema appended for fixed ctrs
        nr_fixed = len(self.fix_registers)
        for i in range(0,nr_fixed):
            dev_schema.append(event_map['FIXED'+str(i)])

        dev_schema_desc = ' '.join(dev_schema) + '\n'

        del self.job.schemas[self.name]
        self.job.get_schema(self.name, dev_schema_desc)

    ## Remap data to human readable schema
    def register(self,host):
        # Build stats without ctl registers
        if self.name not in host.stats: return
        stats = host.stats[self.name]
        dev_stats = dict((str(i), numpy.zeros((len(self.job.times),len(self.ctr_registers)+len(self.fix_registers)),numpy.uint64)) for i in stats.keys())

        for dev, array in stats.iteritems():
            data = dev_stats[dev]
            idx = 0
            for j in self.ctr_registers:
                data[:,idx] = numpy.array(array[:,j], numpy.uint64)
                idx += 1
            for j in self.fix_registers:
                data[:,idx] = numpy.array(array[:,j], numpy.uint64)
                idx += 1

        host.stats[self.name] = dev_stats

def process_job(job):

    if 'intel_snb' in job.schemas:
        snb = reformat_counters(job, 'intel_snb', cpu_event_map)
        for host in job.hosts.itervalues():
            snb.register(host)

    if 'intel_snb_cbo' in job.schemas:
        cbo = reformat_counters(job, 'intel_snb_cbo',cbo_event_map)
        for host in job.hosts.itervalues():
            cbo.register(host)

    if 'intel_snb_hau' in job.schemas:
        hau = reformat_counters(job, 'intel_snb_hau',hau_event_map)
        for host in job.hosts.itervalues():
            hau.register(host)

    if 'intel_snb_imc' in job.schemas:
        imc = reformat_counters(job, 'intel_snb_imc',imc_event_map)
        for host in job.hosts.itervalues():
            imc.register(host)

    if 'intel_snb_qpi' in job.schemas:
        qpi = reformat_counters(job, 'intel_snb_qpi',qpi_event_map)
        for host in job.hosts.itervalues():
            qpi.register(host)

    if 'intel_snb_pcu' in job.schemas:
        pcu = reformat_counters(job, 'intel_snb_pcu',pcu_event_map)
        for host in job.hosts.itervalues():
            pcu.register(host)

    if 'intel_snb_r2pci' in job.schemas:
        r2pci = reformat_counters(job, 'intel_snb_r2pci',r2pci_event_map)
        for host in job.hosts.itervalues():
            r2pci.register(host)

    if 'intel_pmc3' in job.schemas:
        wtm = reformat_counters(job, 'intel_pmc3',wtm_event_map)
        for host in job.hosts.itervalues():
            wtm.register(host)
    if 'intel_uncore' in job.schemas:
        wtmunc = reformat_counters(job, 'intel_uncore',wtmunc_event_map)
        for host in job.hosts.itervalues():
            wtmunc.register(host)
<commit_msg>added Joe's check for mid-job ctl register modifications<commit_after>## @file intel_snb.py
# Post-processing for core and uncore SNB events are performed in this file
# for the pickler.
#
# The registers must be reported in the following order in the stats file:
# -# CTL registers
# -# Programmable CTR registers
# -# Fixed CTR registers
# -# If CTL register programming failed it (QPI) it seems to be 0, so this is added to event maps
# ---->>>> Programmable CTL registers must be in same order as CTR
import numpy

## Processor events
def CORE_PERF_EVENT(event_select, unit_mask):
    return event_select | (unit_mask << 8) | (1L << 16) | (1L << 17) | (1L << 21) | (1L << 22)
## Processor event map
cpu_event_map = {
    CORE_PERF_EVENT(0xD0,0x81) : 'LOAD_OPS_ALL,E',
    CORE_PERF_EVENT(0xD1,0x01) : 'LOAD_OPS_L1_HIT,E',
    CORE_PERF_EVENT(0xD1,0x02) : 'LOAD_OPS_L2_HIT,E',
    CORE_PERF_EVENT(0xD1,0x04) : 'LOAD_OPS_LLC_HIT,E',
    CORE_PERF_EVENT(0x10,0x90) : 'SSE_D_ALL,E',
    CORE_PERF_EVENT(0x11,0x02) : 'SIMD_D_256,E',
    CORE_PERF_EVENT(0xA2,0x01) : 'STALLS,E',
    CORE_PERF_EVENT(0x51,0x01) : 'LOAD_L1D_ALL,E',
    'FIXED0'                   : 'INSTRUCTIONS_RETIRED,E',
    'FIXED1'                   : 'CLOCKS_UNHALTED_CORE,E',
    'FIXED2'                   : 'CLOCKS_UNHALTED_REF,E',
}
## CBo events
def CBOX_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 17) | (0L << 18) | (0L << 19) | (1L << 22) | (0L << 23) | (1L << 24)
## CBo event map
cbo_event_map = {
    CBOX_PERF_EVENT(0x00, 0x00) : 'CLOCK_TICKS,E',
    CBOX_PERF_EVENT(0x11, 0x01) : 'RxR_OCCUPANCY,E',
    CBOX_PERF_EVENT(0x1F, 0x00) : 'COUNTER0_OCCUPANCY,E',
    CBOX_PERF_EVENT(0x34, 0x03) : 'LLC_LOOKUP,E',
    }
## Home Agent Unit events
def HAU_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L << 23) | (1L << 24)
## Home Agent map
hau_event_map = {
    HAU_PERF_EVENT(0x01, 0x03) : 'READ_REQUESTS,E',
    HAU_PERF_EVENT(0x01, 0x0C) : 'WRITE_REQUESTS,E',
    HAU_PERF_EVENT(0x00, 0x00) : 'CLOCKTICKS,E',
    HAU_PERF_EVENT(0x1A, 0x0F) : 'IMC_WRITES,E',
    }
## Integrated Memory events
def IMC_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24)
## Integrated Memory Controller map
imc_event_map = {
    IMC_PERF_EVENT(0x04, 0x03) : 'CAS_READS,E',
    IMC_PERF_EVENT(0x04, 0x0C) : 'CAS_WRITES,E',
    IMC_PERF_EVENT(0x01, 0x00) : 'ACT_COUNT,E',
    IMC_PERF_EVENT(0x02, 0x03) : 'PRE_COUNT_ALL,E',
    IMC_PERF_EVENT(0x02, 0x01) : 'PRE_COUNT_MISS,E',
    'FIXED0'                   : 'CYCLES,E',
    }
## Power Control Unit events
def PCU_PERF_EVENT(event):
    return (event) | (0L << 14) | (0L << 17) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24) | (0L << 31)
## Power Control map
pcu_event_map = {
    PCU_PERF_EVENT(0x06) : 'MAX_OS_CYCLES,E',
    PCU_PERF_EVENT(0x07) : 'MAX_CURRENT_CYCLES,E',
    PCU_PERF_EVENT(0x04) : 'MAX_TEMP_CYCLES,E',
    PCU_PERF_EVENT(0x05) : 'MAX_POWER_CYCLES,E',
    PCU_PERF_EVENT(0x81) : 'MIN_IO_CYCLES,E',
    PCU_PERF_EVENT(0x82) : 'MIN_SNOOP_CYCLES,E',
    'FIXED0'             : 'C3_CYCLES,E',
    'FIXED1'             : 'C6_CYCLES,E',
    }
## QPI Unit events
def QPI_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 15) | (0L << 18) | (1L << 21) | (1L << 22) | (0L <<23) | (1L << 24)
## QPI map
qpi_event_map = {
    QPI_PERF_EVENT(0x00, 0x01) : 'G0_IDLE,E',
    QPI_PERF_EVENT(0x00, 0x04) : 'G0_NON_DATA,E',
    QPI_PERF_EVENT(0x02, 0x08) : 'G1_DRS_DATA,E',
    QPI_PERF_EVENT(0x03, 0x04) : 'G2_NCB_DATA,E',
    }
## R2PCI Unit events
def R2PCI_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (0L << 18) | (1L << 22) | (0L <<23) | (1L << 24)
## R2PCI map
r2pci_event_map = {
    R2PCI_PERF_EVENT(0x24, 0x04) : 'TRANSMITS,E',
    R2PCI_PERF_EVENT(0x01, 0x00) : 'CLOCKTICKS,E',
    R2PCI_PERF_EVENT(0x07, 0x0F) : 'ADDRESS_USED,E',
    R2PCI_PERF_EVENT(0x08, 0x0F) : 'ACKNOWLEDGED_USED,E',
    R2PCI_PERF_EVENT(0x09, 0x0F) : 'DATA_USED,E',
    }
## WESTMERE events
def WTM_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 16) | (1L << 17) | (1L <<21) | (1L << 22)
## WTM map
wtm_event_map = {
    WTM_PERF_EVENT(0x0F, 0x10) : 'MEM_UNCORE_RETIRED_REMOTE_DRAM,E',
    WTM_PERF_EVENT(0x0F, 0x20) : 'MEM_UNCORE_RETIRED_LOCAL_DRAM,E',
    WTM_PERF_EVENT(0x10, 0x01) : 'FP_COMP_OPS_EXE_X87,E',
    WTM_PERF_EVENT(0xCB, 0x01) : 'MEM_LOAD_RETIRED_L1D_HIT,E',
    'FIXED0'                   : 'INSTRUCTIONS_RETIRED,E',
    'FIXED1'                   : 'CLOCKS_UNHALTED_CORE,E',
    'FIXED2'                   : 'CLOCKS_UNHALTED_REF,E',
    }
## WESTMERE UNCORE events
def WTMUNC_PERF_EVENT(event, umask):
    return (event) | (umask << 8) | (1L << 22)
## WTM map
wtmunc_event_map = {
    WTMUNC_PERF_EVENT(0x08, 0x01) : 'L3_HITS_READ,E',
    WTMUNC_PERF_EVENT(0x08, 0x02) : 'L3_HITS_WRITE,E',
    WTMUNC_PERF_EVENT(0x08, 0x04) : 'L3_HITS_PROBE,E',
    WTMUNC_PERF_EVENT(0x09, 0x01) : 'L3_MISS_READ,E',
    WTMUNC_PERF_EVENT(0x09, 0x02) : 'L3_MISS_WRITE,E',
    WTMUNC_PERF_EVENT(0x09, 0x04) : 'L3_MISS_PROBE,E',
    WTMUNC_PERF_EVENT(0x0A, 0x0F) : 'L3_LINES_IN_ANY,E',
    WTMUNC_PERF_EVENT(0x0B, 0x1F) : 'L3_LINES_OUT_ANY,E',
    'FIXED0'                      : 'CLOCKS_UNCORE,E',
    }

## Reformats the counter arrays with configurable events
class reformat_counters:

    ## Constructor
    def __init__(self, job, name, event_map):
        ## Job Object
        self.job = job
        ## Name of device
        self.name = name
        ## List of CTL registers
        self.ctl_registers = []
        ## List of CTR registers
        self.ctr_registers = []
        ## List of Fixed registers
        self.fix_registers = []

        # Just need the first hosts schema
        stats = None
        for host in job.hosts.itervalues():
            if name not in host.stats: return
            stats = host.stats[name]
            break

        if stats == None:
            return

        schema_desc = job.schemas.get(self.name).desc
        registers = schema_desc.split()

        for reg in registers:
            if reg.split(',')[1] != 'C':
                if reg.find('FIXED') == -1:
                    self.ctr_registers.append(registers.index(reg))
                else:
                    self.fix_registers.append(registers.index(reg))
            else:
                if reg.find('GLOBAL') == -1 and reg.find('FIXED') == -1 and reg.find('OPCODE_MATCH') == -1:
                    self.ctl_registers.append(registers.index(reg))

        # Build Schema from ctl registers and event maps
        dev_schema = []
        for dev, array in stats.iteritems():
            for j in self.ctl_registers:
                dev_schema.append(event_map.get(array[0,j],str(array[0,j])))
            break

        # Now check for all hosts:
        # all devices have the same control settings
        # all devices control setting remain the same during the job
        for host in job.hosts.itervalues():
            if name in host.stats:
                for dev, array in host.stats[name].iteritems():
                    devidx = 0
                    for j in self.ctl_registers:
                        settings = array[:,j]
                        if event_map.get(settings[0],str(settings[0])) != dev_schema[devidx] or settings.min() != settings.max():
                            # The control settings for this device changed during the run
                            # mark as the error metric
                            dev_schema[devidx] = "ERROR,E"
                        devidx += 1

        # Schema appended for fixed ctrs
        nr_fixed = len(self.fix_registers)
        for i in range(0,nr_fixed):
            dev_schema.append(event_map['FIXED'+str(i)])

        dev_schema_desc = ' '.join(dev_schema) + '\n'

        del self.job.schemas[self.name]
        self.job.get_schema(self.name, dev_schema_desc)

    ## Remap data to human readable schema
    def register(self,host):
        # Build stats without ctl registers
        if self.name not in host.stats: return
        stats = host.stats[self.name]
        dev_stats = dict((str(i), numpy.zeros((len(self.job.times),len(self.ctr_registers)+len(self.fix_registers)),numpy.uint64)) for i in stats.keys())

        for dev, array in stats.iteritems():
            data = dev_stats[dev]
            idx = 0
            for j in self.ctr_registers:
                data[:,idx] = numpy.array(array[:,j], numpy.uint64)
                idx += 1
            for j in self.fix_registers:
                data[:,idx] = numpy.array(array[:,j], numpy.uint64)
                idx += 1

        host.stats[self.name] = dev_stats

def process_job(job):

    if 'intel_snb' in job.schemas:
        snb = reformat_counters(job, 'intel_snb', cpu_event_map)
        for host in job.hosts.itervalues():
            snb.register(host)

    if 'intel_snb_cbo' in job.schemas:
        cbo = reformat_counters(job, 'intel_snb_cbo',cbo_event_map)
        for host in job.hosts.itervalues():
            cbo.register(host)

    if 'intel_snb_hau' in job.schemas:
        hau = reformat_counters(job, 'intel_snb_hau',hau_event_map)
        for host in job.hosts.itervalues():
            hau.register(host)

    if 'intel_snb_imc' in job.schemas:
        imc = reformat_counters(job, 'intel_snb_imc',imc_event_map)
        for host in job.hosts.itervalues():
            imc.register(host)

    if 'intel_snb_qpi' in job.schemas:
        qpi = reformat_counters(job, 'intel_snb_qpi',qpi_event_map)
        for host in job.hosts.itervalues():
            qpi.register(host)

    if 'intel_snb_pcu' in job.schemas:
        pcu = reformat_counters(job, 'intel_snb_pcu',pcu_event_map)
        for host in job.hosts.itervalues():
            pcu.register(host)

    if 'intel_snb_r2pci' in job.schemas:
        r2pci = reformat_counters(job, 'intel_snb_r2pci',r2pci_event_map)
        for host in job.hosts.itervalues():
            r2pci.register(host)

    if 'intel_pmc3' in job.schemas:
        wtm = reformat_counters(job, 'intel_pmc3',wtm_event_map)
        for host in job.hosts.itervalues():
            wtm.register(host)
    if 'intel_uncore' in job.schemas:
        wtmunc = reformat_counters(job, 'intel_uncore',wtmunc_event_map)
        for host in job.hosts.itervalues():
            wtmunc.register(host)
<|endoftext|>


<commit_before>"""Tests for the models of the ``task_list`` app."""
from django.test import TestCase

from django_libs.tests.factories import UserFactory

from ..models import Task, TaskList
from .factories import (
    CategoryFactory,
    ParentFactory,
    TaskAttachmentFactory,
    TaskFactory,
    TaskListFactory,
)

class CategoryTestCase(TestCase):
    """Tests for the ``Category`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Category`` model."""
        category = CategoryFactory()
        self.assertTrue(category.pk)

class ParentTestCase(TestCase):
    """Tests for the ``Parent`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Parent`` model."""
        parent = ParentFactory()
        self.assertTrue(parent.pk)

class TaskTestCase(TestCase):
    """Tests for the ``Task`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Task`` model."""
        task = TaskFactory()
        self.assertTrue(task.pk)

class TaskAttachmentTestCase(TestCase):
    """Tests for the ``TestAttachment``model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``TaskAttachment`` model."""
        attachment = TaskAttachmentFactory()
        self.assertTrue(attachment.pk)

class TaskListManagerTestCase(TestCase):
    """Tests for the ``TaskListManager`` custom manager."""
    longMessage = True

    def setUp(self):
        self.task_list = TaskListFactory()
        self.task = TaskFactory(task_list=self.task_list)
        self.user = UserFactory()
        self.other_user = UserFactory()
        self.task_list.users.add(self.user, self.other_user)

    def test_manager(self):
        """Tests for the ``TaskListManager`` custom manager."""
        template = TaskList.objects.create_template_from_task_list(
            self.task_list, self.user)
        self.assertEqual(TaskList.objects.all().count(), 2, msg=(
            'After creating a template, there should be 2 task lists in the'
            ' database.'))
        self.assertEqual(Task.objects.all().count(), 2, msg=(
            'After creating a template, there should be 2 tasks in the'
            ' database.'))
        self.assertTrue(template.is_template, msg=(
            'Task list should be a template.'))
        self.assertEqual(template.users.count(), 1, msg=(
            'The template should only have 1 user assigned.'))
        self.assertEqual(template.tasks.count(), 1, msg=(
            'The template should have one task.'))

        task_list = TaskList.objects.create_from_template(
            template, 'new', self.user)
        self.assertEqual(TaskList.objects.all().count(), 3, msg=(
            'After creating a task list, there should be 3 task lists in the'
            ' database.'))
        self.assertEqual(Task.objects.all().count(), 3, msg=(
            'After creating a task list, there should be 3 tasks in the'
            ' database.'))
        self.assertFalse(task_list.is_template, msg=(
            'Task list should not be a template.'))
        self.assertEqual(task_list.users.count(), 1, msg=(
            'The task list should still have 1 user assigned.'))
        self.assertEqual(template.tasks.count(), 1, msg=(
            'The task list should have one task.'))

class TaskListTestCase(TestCase):
    """Tests for the ``TaskList`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``TaskList`` model."""
        task_list = TaskListFactory()
        self.assertTrue(task_list.pk)
<commit_msg>codereview fixes for f8436980d27b07f0a532906641a05a71b125e505<commit_after>"""Tests for the models of the ``task_list`` app."""
from django.test import TestCase

from django_libs.tests.factories import UserFactory

from ..models import Task, TaskList
from .factories import (
    CategoryFactory,
    ParentFactory,
    TaskAttachmentFactory,
    TaskFactory,
    TaskListFactory,
)

class CategoryTestCase(TestCase):
    """Tests for the ``Category`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Category`` model."""
        category = CategoryFactory()
        self.assertTrue(category.pk)

class ParentTestCase(TestCase):
    """Tests for the ``Parent`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Parent`` model."""
        parent = ParentFactory()
        self.assertTrue(parent.pk)

class TaskTestCase(TestCase):
    """Tests for the ``Task`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``Task`` model."""
        task = TaskFactory()
        self.assertTrue(task.pk)

class TaskAttachmentTestCase(TestCase):
    """Tests for the ``TestAttachment``model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``TaskAttachment`` model."""
        attachment = TaskAttachmentFactory()
        self.assertTrue(attachment.pk)

class TaskListManagerTestCase(TestCase):
    """Tests for the ``TaskListManager`` custom manager."""
    longMessage = True

    def setUp(self):
        self.task_list = TaskListFactory()
        self.task = TaskFactory(task_list=self.task_list)
        self.template = TaskListFactory(is_template=True)
        self.template_task = TaskFactory(task_list=self.template)
        self.user = UserFactory()
        self.other_user = UserFactory()
        self.task_list.users.add(self.user, self.other_user)
        self.template.users.add(self.user, self.other_user)

    def test_create_template_from_task_list(self):
        """Tests for the ``create_template_from_task_list`` manager method."""
        template = TaskList.objects.create_template_from_task_list(
            self.task_list, self.user)
        self.assertEqual(TaskList.objects.all().count(), 3, msg=(
            'After creating a template, there should be 2 task lists in the'
            ' database.'))
        self.assertEqual(Task.objects.all().count(), 3, msg=(
            'After creating a template, there should be 2 tasks in the'
            ' database.'))
        self.assertTrue(template.is_template, msg=(
            'Task list should be a template.'))
        self.assertEqual(template.users.count(), 1, msg=(
            'The template should only have 1 user assigned.'))
        self.assertEqual(template.tasks.count(), 1, msg=(
            'The template should have one task.'))

    def create_from_template(self):
        """Tests for the ``create_from_template`` manager method."""
        task_list = TaskList.objects.create_from_template(
            self.template, 'new', self.user)
        self.assertEqual(TaskList.objects.all().count(), 3, msg=(
            'After creating a task list, there should be 3 task lists in the'
            ' database.'))
        self.assertEqual(Task.objects.all().count(), 3, msg=(
            'After creating a task list, there should be 3 tasks in the'
            ' database.'))
        self.assertFalse(task_list.is_template, msg=(
            'Task list should not be a template.'))
        self.assertEqual(task_list.users.count(), 1, msg=(
            'The task list should still have 1 user assigned.'))
        self.assertEqual(task_list.tasks.count(), 1, msg=(
            'The task list should have one task.'))

class TaskListTestCase(TestCase):
    """Tests for the ``TaskList`` model class."""
    longMessage = True

    def test_instantiation(self):
        """Test instantiation of the ``TaskList`` model."""
        task_list = TaskListFactory()
        self.assertTrue(task_list.pk)
<|endoftext|>


<commit_before>from tcga_encoder.utils.helpers import *
from tcga_encoder.data.data import *
from tcga_encoder.definitions.tcga import *
from tcga_encoder.definitions.nn import *
from tcga_encoder.definitions.locations import *
from tcga_encoder.algorithms import *
import seaborn as sns

sns.set_style("whitegrid")
sns.set_context("talk")

pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

#import tensorflow as tf

#load_data_from_dict

#from tensorflow import *

#from models.layers import *
#from models.regularizers import *
#from models.algorithms import *

#from models.vae.tcga_models import *
#from utils.utils import *
#from data.load_datasets_from_broad import load_sources

#from utils.image_utils import *

def add_variables( var_dict, data_dict ):
  # add very specific numbers:
  var_dict["dna_dim"]    = data_dict['dataset'].GetDimension("DNA")
  var_dict["meth_dim"]   = data_dict['dataset'].GetDimension("METH")
  var_dict["rna_dim"]    = data_dict['dataset'].GetDimension("RNA")
  var_dict["mirna_dim"]    = data_dict['dataset'].GetDimension("miRNA")
  var_dict["tissue_dim"] = data_dict['dataset'].GetDimension("TISSUE")

def load_architecture( arch_dict, data_dict ):
  add_variables( arch_dict[VARIABLES], data_dict )
  return arch_dict[NETWORK]( arch_dict, data_dict)

# def load_architectures( arches, data ):
#   networks = OrderedDict()
#   for arch in arches:
#     networks[ arch[NAME] ] = load_architecture( arch, data )
#   return networks

def main(yaml_file):
  y = load_yaml( yaml_file)

  logging_dict = {}
  #print "Loading data"
  load_data_from_dict( y[DATA] )
  algo_dict = y[ALGORITHM]
  arch_dict = y[ARCHITECTURE]
  data_dict = y[DATA] #{N_TRAIN:4000}
  logging_dict = y[LOGGING]
  logging_dict[SAVEDIR] = os.path.join( HOME_DIR, os.path.join( logging_dict[LOCATION], logging_dict[EXPERIMENT] ) )
  # #networks = load_architectures( y[ARCHITECTURES], y[DATA] )
  #add_variables( arch_dict[VARIABLES], data_dict )
  network = load_architecture( arch_dict, data_dict )
  network_name = arch_dict[NAME]

  #

  # make BATCHER and reassign it to dict
  algo_dict[BATCHER] = algo_dict[BATCHER]( network_name, None, data_dict, algo_dict, arch_dict, logging_dict )

  algo_dict[BATCHER].network_name   = network_name
  algo_dict[BATCHER].network        = network

  sess = tf.InteractiveSession()

  results_dict = {}
  train( sess, network, algo_dict, data_dict, logging_dict, results_dict )
  batcher = algo_dict[BATCHER]
  batcher.CloseAll()

######################################################################################################
if __name__ == "__main__":
  assert len(sys.argv) >= 2, "Must pass yaml file."
  yaml_file = sys.argv[1]
  print "Running: ",yaml_file

  main( yaml_file )

  <commit_msg>return batcher, see Z<commit_after>from tcga_encoder.utils.helpers import *
from tcga_encoder.data.data import *
from tcga_encoder.definitions.tcga import *
from tcga_encoder.definitions.nn import *
from tcga_encoder.definitions.locations import *
from tcga_encoder.algorithms import *
import seaborn as sns

sns.set_style("whitegrid")
sns.set_context("talk")

pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

#import tensorflow as tf

#load_data_from_dict

#from tensorflow import *

#from models.layers import *
#from models.regularizers import *
#from models.algorithms import *

#from models.vae.tcga_models import *
#from utils.utils import *
#from data.load_datasets_from_broad import load_sources

#from utils.image_utils import *

def add_variables( var_dict, data_dict ):
  # add very specific numbers:
  var_dict["dna_dim"]    = data_dict['dataset'].GetDimension("DNA")
  var_dict["meth_dim"]   = data_dict['dataset'].GetDimension("METH")
  var_dict["rna_dim"]    = data_dict['dataset'].GetDimension("RNA")
  var_dict["mirna_dim"]    = data_dict['dataset'].GetDimension("miRNA")
  var_dict["tissue_dim"] = data_dict['dataset'].GetDimension("TISSUE")

def load_architecture( arch_dict, data_dict ):
  add_variables( arch_dict[VARIABLES], data_dict )
  return arch_dict[NETWORK]( arch_dict, data_dict)

# def load_architectures( arches, data ):
#   networks = OrderedDict()
#   for arch in arches:
#     networks[ arch[NAME] ] = load_architecture( arch, data )
#   return networks

def main(yaml_file):
  y = load_yaml( yaml_file)

  logging_dict = {}
  #print "Loading data"
  load_data_from_dict( y[DATA] )
  algo_dict = y[ALGORITHM]
  arch_dict = y[ARCHITECTURE]
  data_dict = y[DATA] #{N_TRAIN:4000}
  logging_dict = y[LOGGING]
  logging_dict[SAVEDIR] = os.path.join( HOME_DIR, os.path.join( logging_dict[LOCATION], logging_dict[EXPERIMENT] ) )
  # #networks = load_architectures( y[ARCHITECTURES], y[DATA] )
  #add_variables( arch_dict[VARIABLES], data_dict )
  network = load_architecture( arch_dict, data_dict )
  network_name = arch_dict[NAME]

  #

  # make BATCHER and reassign it to dict
  algo_dict[BATCHER] = algo_dict[BATCHER]( network_name, None, data_dict, algo_dict, arch_dict, logging_dict )

  algo_dict[BATCHER].network_name   = network_name
  algo_dict[BATCHER].network        = network

  sess = tf.InteractiveSession()

  results_dict = {}
  train( sess, network, algo_dict, data_dict, logging_dict, results_dict )
  batcher = algo_dict[BATCHER]
  batcher.CloseAll()
  return batcher

######################################################################################################
if __name__ == "__main__":
  assert len(sys.argv) >= 2, "Must pass yaml file."
  yaml_file = sys.argv[1]
  print "Running: ",yaml_file

  batcher = main( yaml_file )
  fill = batcher.fill_source_store
  fill.open()
  Z_train = fill["/Z/TRAIN/Z/mu"].loc[ batcher.train_barcodes]
  Z_val = fill["/Z/VAL/Z/mu"].loc[ batcher.validation_barcodes]
  fill.close()

  <|endoftext|>


<commit_before>from abc import abstractmethod, abstractproperty

from ledger.util import count_bits_set
from ledger.util import highest_bit_set

class HashStore:
    """
    Store of nodeHashes and leafHashes mapped against their sequence numbers.
    """

    @abstractmethod
    def writeLeaf(self, leafHash):
        """
        append the leafHash to the leaf hash store

        :param leafHash: hash of the leaf
        """
        pass

    @abstractmethod
    def writeNode(self, node):
        """
        append the node to the node hash store.

        :param node: tuple of start, height and nodeHash
        """
        pass

    @abstractmethod
    def readLeaf(self, pos):
        """
        Read the leaf hash at the given position in the merkle tree

        :param pos: the sequence number of the leaf
        :return: the leafHash at the specified position
        """
        pass

    @abstractmethod
    def readNode(self, pos):
        """
        Read the node hash at the given position in the merkle tree

        :param pos: the sequence number of the node (as calculated by
        getNodePosition)
        :return: the nodeHash at the specified position
        """
        pass

    @abstractmethod
    def readLeafs(self, startpos, endpos):
        """
        Read multiple leaves at the given position.

        :param startpos: read from this sequence number (inclusive)
        :param endpos: read up to this sequence number (inclusive)
        :return: list of leafHashes
        """
        pass

    @abstractmethod
    def readNodes(self, startpos, endpos):
        """
        Read multiple nodes at the given position. Node position can be
         calculated using getNodePosition

        :param startpos: read from this sequence number (inclusive)
        :param endpos: read up to this sequence number (inclusive)
        :return: list of nodeHashes
        """
        pass

    @abstractproperty
    def leafCount(self) -> int:
        pass

    @leafCount.setter
    @abstractmethod
    def leafCount(self, count: int) -> None:
        pass

    @classmethod
    def getNodePosition(cls, start, height=None) -> int:
        """
        Calculates node position based on start and height

        :param start: The sequence number of the first leaf under this tree.
        :param height: Height of this node in the merkle tree
        :return: the node's position
        """
        pwr = highest_bit_set(start) - 1
        height = height or pwr
        if count_bits_set(start) == 1:
            adj = height - pwr
            return start - 1 + adj
        else:
            c = pow(2, pwr)
            return cls.getNodePosition(c, pwr) + \
                   cls.getNodePosition(start - c, height)

    @classmethod
    def getPath(cls, seqNo, offset=0):
        """
        Get the audit path of the leaf at the position specified by serNo.

        :param seqNo: sequence number of the leaf to calculate the path for
        :param offset: the sequence number of the node from where the path
         should begin.
        :return: tuple of leafs and nodes
        """
        if offset >= seqNo:
            raise ValueError("Offset should be less than serial number")
        pwr = highest_bit_set(seqNo - 1 - offset) - 1
        if pwr <= 0:
            if seqNo % 2 == 0:
                return [seqNo - 1], []
            else:
                return [], []
        c = pow(2, pwr) + offset
        leafs, nodes = cls.getPath(seqNo, c)
        nodes.append(cls.getNodePosition(c, pwr))
        return leafs, nodes

    def readNodeByTree(self, start, height=None):
        """
        Fetches nodeHash based on start leaf and height of the node in the tree.

        :return: the nodeHash
        """
        pos = self.getNodePosition(start, height)
        return self.readNode(pos)
<commit_msg>add nodeCount property to HashStore<commit_after>from abc import abstractmethod, abstractproperty

from ledger.util import count_bits_set
from ledger.util import highest_bit_set

class HashStore:
    """
    Store of nodeHashes and leafHashes mapped against their sequence numbers.
    """

    @abstractmethod
    def writeLeaf(self, leafHash):
        """
        append the leafHash to the leaf hash store

        :param leafHash: hash of the leaf
        """
        pass

    @abstractmethod
    def writeNode(self, node):
        """
        append the node to the node hash store.

        :param node: tuple of start, height and nodeHash
        """
        pass

    @abstractmethod
    def readLeaf(self, pos):
        """
        Read the leaf hash at the given position in the merkle tree

        :param pos: the sequence number of the leaf
        :return: the leafHash at the specified position
        """
        pass

    @abstractmethod
    def readNode(self, pos):
        """
        Read the node hash at the given position in the merkle tree

        :param pos: the sequence number of the node (as calculated by
        getNodePosition)
        :return: the nodeHash at the specified position
        """
        pass

    @abstractmethod
    def readLeafs(self, startpos, endpos):
        """
        Read multiple leaves at the given position.

        :param startpos: read from this sequence number (inclusive)
        :param endpos: read up to this sequence number (inclusive)
        :return: list of leafHashes
        """
        pass

    @abstractmethod
    def readNodes(self, startpos, endpos):
        """
        Read multiple nodes at the given position. Node position can be
         calculated using getNodePosition

        :param startpos: read from this sequence number (inclusive)
        :param endpos: read up to this sequence number (inclusive)
        :return: list of nodeHashes
        """
        pass

    @abstractproperty
    def leafCount(self) -> int:
        pass

    @leafCount.setter
    @abstractmethod
    def leafCount(self, count: int) -> None:
        pass

    @abstractproperty
    def nodeCount(self) -> None:
        pass

    @classmethod
    def getNodePosition(cls, start, height=None) -> int:
        """
        Calculates node position based on start and height

        :param start: The sequence number of the first leaf under this tree.
        :param height: Height of this node in the merkle tree
        :return: the node's position
        """
        pwr = highest_bit_set(start) - 1
        height = height or pwr
        if count_bits_set(start) == 1:
            adj = height - pwr
            return start - 1 + adj
        else:
            c = pow(2, pwr)
            return cls.getNodePosition(c, pwr) + \
                   cls.getNodePosition(start - c, height)

    @classmethod
    def getPath(cls, seqNo, offset=0):
        """
        Get the audit path of the leaf at the position specified by serNo.

        :param seqNo: sequence number of the leaf to calculate the path for
        :param offset: the sequence number of the node from where the path
         should begin.
        :return: tuple of leafs and nodes
        """
        if offset >= seqNo:
            raise ValueError("Offset should be less than serial number")
        pwr = highest_bit_set(seqNo - 1 - offset) - 1
        if pwr <= 0:
            if seqNo % 2 == 0:
                return [seqNo - 1], []
            else:
                return [], []
        c = pow(2, pwr) + offset
        leafs, nodes = cls.getPath(seqNo, c)
        nodes.append(cls.getNodePosition(c, pwr))
        return leafs, nodes

    def readNodeByTree(self, start, height=None):
        """
        Fetches nodeHash based on start leaf and height of the node in the tree.

        :return: the nodeHash
        """
        pos = self.getNodePosition(start, height)
        return self.readNode(pos)
<|endoftext|>


<commit_before>from itertools import chain

from django.shortcuts import render, get_object_or_404
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views import View
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin

from diventi.products.models import Product
from .models import Book, Chapter, Section

class UserHasProductMixin(UserPassesTestMixin):
    """
        This view checks if the user has bought the product
        related to the requested book.
        It assumes to have the slug of the book object available
        in book_slug get parameter.
    """

    permission_denied_message = _('This book is not in your collection, please check your profile.')

    def test_func(self):
        book_slug = self.kwargs.get('book_slug', None)
        book = get_object_or_404(Book, slug=book_slug)
        product = book.book_product
        test = product.user_has_already_bought(self.request.user) or product.user_has_authored(self.request.user)
        return test

class EbookView(View):
    """
        Generic view that manages context data for ebooks.
        It assumes to have the slug of the book object available
        in book_slug get parameter.
    """

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        book_slug = self.kwargs.get('book_slug', None)
        book = get_object_or_404(Book, slug=book_slug)
        context['book'] = book
        chapters = Chapter.objects.filter(chapter_book__slug=book_slug)
        chapters = chapters.order_by('order_index')
        context['chapters'] = chapters
        return context

class BookDetailView(LoginRequiredMixin, UserHasProductMixin,
                     EbookView, DetailView):
    """ Returns the digital content of a product. """

    model = Book
    template_name = "ebooks/book_detail.html"
    slug_url_kwarg = 'book_slug'

    def get_queryset(self, **kwargs):
        queryset = super().get_queryset(**kwargs)
        return queryset.published().product()

class ChapterDetailView(LoginRequiredMixin, UserHasProductMixin,
                        EbookView, DetailView):
    """ Returns the chapter of a book. """

    model = Chapter
    template_name = "ebooks/chapter_detail.html"
    slug_url_kwarg = 'chapter_slug'

    def get_queryset(self, **kwargs):
        queryset = super().get_queryset(**kwargs)
        return queryset.published().sections()

class SectionSearchView(LoginRequiredMixin, UserHasProductMixin,
                        EbookView, ListView):
    """
        Returns a list of sections that matches the searched phrase.
    """
    template_name = "ebooks/search_results.html"
    context_object_name = 'results'
    model = Section

    def get_queryset(self, **kwargs):
        results = super().get_queryset(**kwargs)
        query = self.request.GET.get('q')
        book_slug = self.kwargs.get('book_slug')
        if query:
            sections = Section.search(self, query, book_slug)
            results = list(chain(sections,))
        else:
            results = None
        return results

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['search_query'] = self.request.GET.get('q')
        return context

<commit_msg>Adjusted the ordering of sections in the sidebar.<commit_after>from itertools import chain

from django.shortcuts import render, get_object_or_404
from django.views.generic.detail import DetailView
from django.views.generic.list import ListView
from django.views import View
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin

from diventi.products.models import Product
from .models import Book, Chapter, Section

class UserHasProductMixin(UserPassesTestMixin):
    """
        This view checks if the user has bought the product
        related to the requested book.
        It assumes to have the slug of the book object available
        in book_slug get parameter.
    """

    permission_denied_message = _('This book is not in your collection, please check your profile.')

    def test_func(self):
        book_slug = self.kwargs.get('book_slug', None)
        book = get_object_or_404(Book, slug=book_slug)
        product = book.book_product
        test = product.user_has_already_bought(self.request.user) or product.user_has_authored(self.request.user)
        return test

class EbookView(View):
    """
        Generic view that manages context data for ebooks.
        It assumes to have the slug of the book object available
        in book_slug get parameter.
    """

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        book_slug = self.kwargs.get('book_slug', None)
        book = get_object_or_404(Book, slug=book_slug)
        context['book'] = book
        chapters = Chapter.objects.filter(chapter_book__slug=book_slug)
        chapters = chapters.order_by('order_index').sections()
        context['chapters'] = chapters
        return context

class BookDetailView(LoginRequiredMixin, UserHasProductMixin,
                     EbookView, DetailView):
    """ Returns the digital content of a product. """

    model = Book
    template_name = "ebooks/book_detail.html"
    slug_url_kwarg = 'book_slug'

    def get_queryset(self, **kwargs):
        queryset = super().get_queryset(**kwargs)
        return queryset.published().product()

class ChapterDetailView(LoginRequiredMixin, UserHasProductMixin,
                        EbookView, DetailView):
    """ Returns the chapter of a book. """

    model = Chapter
    template_name = "ebooks/chapter_detail.html"
    slug_url_kwarg = 'chapter_slug'

    def get_queryset(self, **kwargs):
        queryset = super().get_queryset(**kwargs)
        return queryset.published().sections()

class SectionSearchView(LoginRequiredMixin, UserHasProductMixin,
                        EbookView, ListView):
    """
        Returns a list of sections that matches the searched phrase.
    """
    template_name = "ebooks/search_results.html"
    context_object_name = 'results'
    model = Section

    def get_queryset(self, **kwargs):
        results = super().get_queryset(**kwargs)
        query = self.request.GET.get('q')
        book_slug = self.kwargs.get('book_slug')
        if query:
            sections = Section.search(self, query, book_slug)
            results = list(chain(sections,))
        else:
            results = None
        return results

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['search_query'] = self.request.GET.get('q')
        return context

<|endoftext|>